使用 AWS Sagemaker 训练因子分解机模型并应用于推荐系统

df_all = pd.read_csv('all_movie_rates_noblank.csv')print(df_all.shape)print(df_all['UserId'].nunique())print(df_all['MovieID'].nunique())

df=pd.read_csv('all_movie_rates_100k.csv')print(df.shape)print(df['UserId'].nunique())print(df['MovieID'].nunique())
df_train, df_test = train_test_split(df, test_size=0.2, random_state=42)print(df_train.shape, df_test.shape)
print(df_train.head(10))print(df_test.head(10))(107029, 3)

filename = 'all_movie_rates_100k.csv'user_number = 53902movie_number = 26004
def createIDtoIndexDict(filename, user_number, movie_number):    u_Dict = {}    m_Dict = {}    i = 0    m = 0    with open(filename, 'r') as f:        sample = csv.reader(f, delimiter=',')        for MovieID, Rate, UserId in sample:            if UserId == 'UserId':                continue            else:                if UserId not in u_Dict.keys():                    u_Dict[UserId] = i                    i = i+1                 if MovieID not in m_Dict.keys():                    m_Dict[MovieID] = m                    m = m+1    return u_Dict, m_Dict u_Dict, m_Dict = createIDtoIndexDict(filename, user_number, movie_number)

df_train.to_csv('100k_train.csv', index=False, encoding="utf_8_sig")df_test.to_csv('100k_test.csv', index=False, encoding="utf_8_sig")
columns = user_number+movie_numberdef loadDataset(filename, lines, columns):    X = scipy.sparse.lil_matrix((lines, columns)).astype('float32')    Y = []    line = 0    with open (filename, 'r') as f:        sample = csv.reader(f, delimiter=',')        for MovieID, Rate, UserId in sample:            if UserId == 'UserId':                continue            else:                X[line, u_Dict[UserId]] = 1                X[line, user_number+m_Dict[MovieID]] = 1                if Rate == 'Rate' or int(Rate) < 6:                    Y.append(0)                else:                    Y.append(1)                line=line+1
    Y=np.array(Y).astype('float32')    print(X.shape)    print(Y.shape)    return X, Y

X_train, Y_train = loadDataset('100k_train.csv', df_train.shape[0], columns)X_test, Y_test = loadDataset('100k_test.csv', df_test.shape[0], columns)print(X_train.shape, X_test.shape)print(Y_train.shape, Y_test.shape)

bucket = 'movie-recommendation-demo-raw-data'prefix = 'sagemaker/movie-recommendation'train_key      = 'train.protobuf'train_prefix   = '{}/{}'.format(prefix, 'train')test_key       = 'test.protobuf'test_prefix    = '{}/{}'.format(prefix, 'test')output_prefix  = 's3://{}/{}/output'.format(bucket, prefix)
def writeDatasetToProtobuf(X, Y, bucket, prefix, key):    import io,boto3    import sagemaker.amazon.common as smac    buf = io.BytesIO()    smac.write_spmatrix_to_sparse_tensor(buf, X, Y)    buf.seek(0)    print(buf)    obj = '{}/{}'.format(prefix, key)    boto3.resource('s3').Bucket(bucket).Object(obj).upload_fileobj(buf)    print('Wrote dataset: {}/{}'.format(bucket,obj))    return 's3://{}/{}'.format(bucket,obj)    train_data = writeDatasetToProtobuf(X_train, Y_train, bucket, train_prefix, train_key)    test_data = writeDatasetToProtobuf(X_test, Y_test, bucket, test_prefix, test_key)    
print(train_data)print(test_data)print('Output: {}'.format(output_prefix))<_io.BytesIO object at 0x7f99a8b8e1a8>
Wrote dataset: movie-recommendation-demo-raw-data/sagemaker/movie-recommendation/train/train.protobuf
<_io.BytesIO object at 0x7f99a8b8e1a8>
Wrote dataset: movie-recommendation-demo-raw-data/sagemaker/movie-recommendation/test/test.protobuf
s3://movie-recommendation-demo-raw-data/sagemaker/movie-recommendation/train/train.protobuf
s3://movie-recommendation-demo-raw-data/sagemaker/movie-recommendation/test/test.protobuf
Output: s3://movie-recommendation-demo-raw-data/sagemaker/movie-recommendation/output

from sagemaker import get_execution_rolefrom sagemaker.amazon.amazon_estimator import get_image_uriimport sagemakersess = sagemaker.Session()
role =  get_execution_role()container = get_image_uri(boto3.Session().region_name, 'factorization-machines')

fm = sagemaker.estimator.Estimator(container,                                   role,                                    train_instance_count=1,                                    train_instance_type='ml.c4.xlarge',                                   output_path=output_prefix,                                   sagemaker_session=sess)fm.set_hyperparameters(feature_dim=79906,                      predictor_type='binary_classifier',                      mini_batch_size=1000,                      num_factors=64,                      epochs=50)
fm.fit({'train': train_data, 'test':test_data})

#metrics {“Metrics”: {“totaltime”: {“count”: 1, “max”: 40280.484199523926, “sum”: 40280.484199523926, “min”: 40280.484199523926}, “setuptime”: {“count”: 1, “max”: 39.59202766418457, “sum”: 39.59202766418457, “min”: 39.59202766418457}}, “EndTime”: 1545103329.767301, “Dimensions”: {“Host”: “algo-1”, “Operation”: “training”, “Algorithm”: “factorization-machines”}, “StartTime”: 1545103329.299489}

fm.set_hyperparameters(feature_dim=79906,                      predictor_type='binary_classifier',                      mini_batch_size=200,                      num_factors=64,                      epochs=134)In [228]:
fm_predictor = fm.deploy(initial_instance_count=1,                         instance_type='ml.t2.medium')INFO:sagemaker:Creating model with name: factorization-machines-2018-12-18-05-56-31-040
INFO:sagemaker:Creating endpoint with name factorization-machines-2018-12-18-05-47-26-108
—————————————————————–!