使用 AWS Sagemaker 部署的终端节点进行推荐预测的常用场景

#select 100020528 as a random userfor k in df[df['UserId']=='100020528'].index:     print(df_movie_all_db[df_movie_all_db['MovieID']==str(df.at[k,'MovieID'])].Title.values[0], "         ", df.at[k,'Rate'])

#revise the loadDataset() to accept the userID as an input.def createInferenceForUser(UserId, lines, columns):    user_number = 53902    X = scipy.sparse.lil_matrix((lines, columns)).astype('float32')    line = 0    for line in range(0, lines):        X[line, u_Dict[UserId]] = 1        X[line, user_number+line] = 1        line=line+1
    print(X.shape)    return X

user_inference = createInferenceForUser('100020528', movie_number, user_number+movie_number)print(user_inference.shape)

#invoke endpointclient = boto3.client('sagemaker-runtime')In [20]:
#Make inferenceimport jsoninf_start = 0step = 10response = []while inf_start <= user_inference.shape[0]+1:    raw_response = client.invoke_endpoint(                EndpointName = 'factorization-machines-2018-12-18-05-47-26-108',                Body = fm_serializer(user_inference[inf_start:inf_start+step].toarray()),                ContentType='application/json')    result = json.loads(raw_response['Body'].read())    for counter, p in enumerate(result['predictions']):        p['MovieID'] = df_m_Dict[df_m_Dict.movie_index == (inf_start + int(counter))].MovieID.values[0]        response.append(p)    inf_start = inf_start+step    print(inf_start)print(len(response))26004

df_response_like = df_response[df_response.predicted_label == 1.0]final = df_response_like.sort_values('score', ascending=False).head(10)for final_index in final.index:    print(df_movie_all_db[df_movie_all_db['MovieID']==str(final.at[final_index,'MovieID'])].Title.values[0],'\t',df_movie_all_db[df_movie_all_db['MovieID']==str(final.at[final_index,'MovieID'])].Rate.values[0],'\t',final.at[final_index,'score'])

def createNewUserInference(lines, columns):    user_number = 53902    X = scipy.sparse.lil_matrix((lines, columns)).astype('float32')    line = 0    for line in range(0, lines):        X[line, user_number+line] = 1        line=line+1    print(X.shape)    return X

def callEndpointInference(query):    import json    inf_start = 0    step = 10    response = []    while inf_start <= query.shape[0]:        raw_response = client.invoke_endpoint(                    EndpointName = 'factorization-machines-2018-12-18-05-47-26-108',                    Body = fm_serializer(query[inf_start:min(inf_start+step, query.shape[0]+1)].toarray()),                    ContentType='application/json')        result = json.loads(raw_response['Body'].read())        for counter, p in enumerate(result['predictions']):            p['MovieID'] = df_m_Dict[df_m_Dict.movie_index == (inf_start + int(counter))].MovieID.values[0]            response.append(p)        inf_start = inf_start+step        print(inf_start)    return response

def showResult(response):    df_response = pd.DataFrame(response)    df_response_like = df_response[df_response.predicted_label == 1.0]    final = df_response_like.sort_values('score', ascending=False).head(10)    for final_index in final.index:        print(df_movie_all_db[df_movie_all_db['MovieID']==str(final.at[final_index,'MovieID'])].Title.values[0],'\t',df_movie_all_db[df_movie_all_db['MovieID']==str(final.at[final_index,'MovieID'])].Rate.values[0],'\t',final.at[final_index,'score'])

new_inference = createNewUserInference(movie_number, user_number+movie_number)(26004, 79906)

%%timenew_response = callEndpointInference(new_inference) 
CPU times: user 5min 18s, sys: 25.9 s, total: 5min 44s
Wall time: 16min

showResult(new_response)

df_movie_preference = df_movie_all_db[df_movie_all_db['MovieID'].isin(movie_rated_by_user)].copy()df_movie_preference.groupby('Tags').count()

df_movie_all_db[df_movie_all_db.Tags.isin(['惊悚','恐怖'])].count() 

#Check whether all 1625 movies is in m_Dict, and clean the movies that are not in m_Dictfor m_h in horror_and_terrified_movie:    if m_h in m_Dict.keys():        continue    else:        print(m_h)        horror_and_terrified_movie.remove(m_h)print(len(horror_and_terrified_movie))1124

m_rated = []for el_index in df_movie_preference.index:    m_rated.append(df_movie_preference.at[el_index, 'MovieID'])

for m_h in horror_and_terrified_movie:    if m_h in m_rated:        print(m_h)        horror_and_terrified_movie.remove(m_h)print(len(horror_and_terrified_movie))1104

filtered_user_inference = createInferenceForUserPreference('100020528', horror_and_terrified_movie, user_number+movie_number)print(filtered_user_inference.shape)(1104, 79906)

%%timefiltered_response = callEndpointInference_noid(filtered_user_inference)CPU times: user 13.2 s, sys: 879 ms, total: 14.1 s
Wall time: 40.9 s

showResultfromDataFrame(df_filtered_counter)

def slicedInferenceUserPreference(userid, step, movielist):    import json    #total_line = 26004 #movie_number        start = 0    response = []    line = 0    response = []        while start <= len(movielist)+1:        js={'instances': []}        for line in range(0, min(int(step), len(movielist)-start)):            js['instances'].append({'data':{'features': {                'keys':[int(u_Dict[userid]), 53902+m_Dict[movielist[line+int(start)]]],                "shape":[79906],                "values":[1,1]}}})                #print(json.dumps(js))        raw_response = client.invoke_endpoint(                EndpointName = 'factorization-machines-2018-12-18-05-47-26-108',                Body = json.dumps(js),                ContentType='application/json')        result = json.loads(raw_response['Body'].read())        for counter, p in enumerate(result['predictions']):            p['MovieID'] = movielist[counter]            response.append(p)        print(start+line)        start = start+step        return(response)In [299]:
%%timetest_user_preference_response = slicedInferenceUserPreference('100020528', 1000, horror_and_terrified_movie)999
1103
CPU times: user 21.8 ms, sys: 519 µs, total: 22.3 ms
Wall time: 5.93 s

showResultfromDataFrame(preference_response)