CF Stacker
Introduction¶
Contining on Part 3 of this demo series, we will focus on CF ensemble as a stacker on top of other ensemble learning methods; e.g., stacking upon stacking.
- Stacking upon stacking works by using a meta-learner such as logistic regression to make an initial label prediciton on
Tfollowed by CF ensemble method to reestimate probabilities. - Motivation: CF methods are greatly affected by the labeling guess on the test split (
T)- Majority vote does not always result in "sufficient" accuracy for TPs, leading to a degradation of the "qualify" of the reestimated matrices, where the quality in this context is evaluated by how well the reestimatation can be used to predict the test or unseen data just like the regular classification settings.
- kNN-based mehtods do not always perform well either
Reference¶
In [ ]:
Copied!
import warnings
warnings.filterwarnings('ignore')
import numpy as np
import pandas as pd
from pandas import DataFrame, Series
import os, sys
# Colab
try:
import google.colab
IN_COLAB = True
except:
IN_COLAB = False
# Plotting
import matplotlib.pylab as plt
# %matplotlib inline
from matplotlib.pyplot import figure
import seaborn as sns
from IPython.display import display
# Progress
from tqdm import tqdm
################################################################
# Configure system environment
# - Please modify input_dir according to your local enviornment
#
################################################################
cur_dir = os.getcwd()
project_dir = 'machine_learning_examples/cf_ensemble'
if IN_COLAB:
# Run this demo on Google Colab
from google.colab import drive
drive.mount('/content/drive')
# Parameters for data
input_dir = f"/content/drive/MyDrive/Colab Notebooks/{project_dir}"
# /content/drive/MyDrive/Colab Notebooks/machine_learning_examples/data/data-is-life
sys.path.append(input_dir)
else:
input_dir = cur_dir
if input_dir != cur_dir:
sys.path.append(input_dir)
print(f"> Adding {input_dir} to sys path ...")
print(sys.path)
import warnings
warnings.filterwarnings('ignore')
import numpy as np
import pandas as pd
from pandas import DataFrame, Series
import os, sys
# Colab
try:
import google.colab
IN_COLAB = True
except:
IN_COLAB = False
# Plotting
import matplotlib.pylab as plt
# %matplotlib inline
from matplotlib.pyplot import figure
import seaborn as sns
from IPython.display import display
# Progress
from tqdm import tqdm
################################################################
# Configure system environment
# - Please modify input_dir according to your local enviornment
#
################################################################
cur_dir = os.getcwd()
project_dir = 'machine_learning_examples/cf_ensemble'
if IN_COLAB:
# Run this demo on Google Colab
from google.colab import drive
drive.mount('/content/drive')
# Parameters for data
input_dir = f"/content/drive/MyDrive/Colab Notebooks/{project_dir}"
# /content/drive/MyDrive/Colab Notebooks/machine_learning_examples/data/data-is-life
sys.path.append(input_dir)
else:
input_dir = cur_dir
if input_dir != cur_dir:
sys.path.append(input_dir)
print(f"> Adding {input_dir} to sys path ...")
print(sys.path)
Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount("/content/drive", force_remount=True).
> Adding /content/drive/MyDrive/Colab Notebooks/machine_learning_examples/cf_ensemble to sys path ...
['', '/content', '/env/python', '/usr/lib/python37.zip', '/usr/lib/python3.7', '/usr/lib/python3.7/lib-dynload', '/usr/local/lib/python3.7/dist-packages', '/usr/lib/python3/dist-packages', '/usr/local/lib/python3.7/dist-packages/IPython/extensions', '/root/.ipython', '/content/drive/MyDrive/Colab Notebooks/machine_learning_examples/cf_ensemble', '/content/drive/MyDrive/Colab Notebooks/machine_learning_examples/cf_ensemble']
In [ ]:
Copied!
# Tensorflow
import tensorflow as tf
print(tf.__version__)
# import tensorflow_probability as tfp
# tfd = tfp.distributions
from tensorflow import keras
# from tensorflow.keras import layers
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, Lambda, Embedding
from tensorflow.keras.optimizers import RMSprop
from keras.utils.vis_utils import plot_model
from tensorflow.keras import backend as K
#################################################################
# Scikit-learn
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold, cross_val_predict, cross_val_score
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, StackingClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
#################################################################
# CF-ensemble-specific libraries
import utils_stacking as ustk
import utils_classifier as uclf
import utils_sys as usys
import utils_cf as uc
import polarity_models as pmodel
from polarity_models import Polarity
import scipy.sparse as sparse
from utils_sys import highlight
#################################################################
# Misc
import pprint
import tempfile
from typing import Dict, Text
np.set_printoptions(precision=3, edgeitems=5, suppress=True)
# Tensorflow
import tensorflow as tf
print(tf.__version__)
# import tensorflow_probability as tfp
# tfd = tfp.distributions
from tensorflow import keras
# from tensorflow.keras import layers
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, Lambda, Embedding
from tensorflow.keras.optimizers import RMSprop
from keras.utils.vis_utils import plot_model
from tensorflow.keras import backend as K
#################################################################
# Scikit-learn
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold, cross_val_predict, cross_val_score
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, StackingClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
#################################################################
# CF-ensemble-specific libraries
import utils_stacking as ustk
import utils_classifier as uclf
import utils_sys as usys
import utils_cf as uc
import polarity_models as pmodel
from polarity_models import Polarity
import scipy.sparse as sparse
from utils_sys import highlight
#################################################################
# Misc
import pprint
import tempfile
from typing import Dict, Text
np.set_printoptions(precision=3, edgeitems=5, suppress=True)
2.8.0
Generating training data¶
In [ ]:
Copied!
%matplotlib inline
import data_pipeline as dp
max_class_ratio=0.99
# get the dataset
X0, y0 = dp.generate_imbalanced_data(class_ratio=max_class_ratio, verbose=1)
%matplotlib inline
import data_pipeline as dp
max_class_ratio=0.99
# get the dataset
X0, y0 = dp.generate_imbalanced_data(class_ratio=max_class_ratio, verbose=1)
> n_classes: 2
[0 1]
> counts:
Counter({0: 4465, 1: 535})
Choosing base classifiers¶
In [ ]:
Copied!
# Create Base Learners
base_learners = [
('RF', RandomForestClassifier(n_estimators= 200,
oob_score = True,
class_weight = "balanced",
random_state = 20,
ccp_alpha = 0.1)),
('KNNC', KNeighborsClassifier(n_neighbors = len(np.unique(y0))
, weights = 'distance')),
# ('SVC', SVC(kernel = 'linear', probability=True,
# class_weight = 'balanced'
# , break_ties = True)),
('GNB', GaussianNB()),
('QDA', QuadraticDiscriminantAnalysis()),
('MLPClassifier', MLPClassifier(alpha=1, max_iter=1000)),
# ('DT', DecisionTreeClassifier(max_depth=5)),
# ('GPC', GaussianProcessClassifier(1.0 * RBF(1.0))),
]
# Create Base Learners
base_learners = [
('RF', RandomForestClassifier(n_estimators= 200,
oob_score = True,
class_weight = "balanced",
random_state = 20,
ccp_alpha = 0.1)),
('KNNC', KNeighborsClassifier(n_neighbors = len(np.unique(y0))
, weights = 'distance')),
# ('SVC', SVC(kernel = 'linear', probability=True,
# class_weight = 'balanced'
# , break_ties = True)),
('GNB', GaussianNB()),
('QDA', QuadraticDiscriminantAnalysis()),
('MLPClassifier', MLPClassifier(alpha=1, max_iter=1000)),
# ('DT', DecisionTreeClassifier(max_depth=5)),
# ('GPC', GaussianProcessClassifier(1.0 * RBF(1.0))),
]
Load pre-trained level-1 data¶
- If it's unclear how to obtain the pre-trained dataset (e.g. probability matrices from base classifiers), please refer back to part 1 or part 2 of this demo series.
In [ ]:
Copied!
import cf_models as cm
tLoadPretrained = False
######################
fold_number = 0
n_iterations = 1
data_dir = os.path.join(input_dir, 'data')
######################
if not tLoadPretrained:
# Use the previously selected base predictors (`base_learners`) to generate the level-1 dataset
R, T, U, L_train, L_test = cm.demo_cf_stacking(input_data=(X0, y0),
input_dir=input_dir, n_iter=n_iterations,
base_learners=base_learners, # <<< base classifiers selected
verbose=1)
else:
R, T, U, L_train, L_test = dp.load_pretrained_level1_data(fold_number=fold_number, verbose=1, data_dir=data_dir)
# Derived quantities
n_train = R.shape[1]
p_threshold = uc.estimateProbThresholds(R, L=L_train, pos_label=1, policy='fmax')
import cf_models as cm
tLoadPretrained = False
######################
fold_number = 0
n_iterations = 1
data_dir = os.path.join(input_dir, 'data')
######################
if not tLoadPretrained:
# Use the previously selected base predictors (`base_learners`) to generate the level-1 dataset
R, T, U, L_train, L_test = cm.demo_cf_stacking(input_data=(X0, y0),
input_dir=input_dir, n_iter=n_iterations,
base_learners=base_learners, # <<< base classifiers selected
verbose=1)
else:
R, T, U, L_train, L_test = dp.load_pretrained_level1_data(fold_number=fold_number, verbose=1, data_dir=data_dir)
# Derived quantities
n_train = R.shape[1]
p_threshold = uc.estimateProbThresholds(R, L=L_train, pos_label=1, policy='fmax')
2.8.0
0%| | 0/1 [00:00<?, ?it/s]
(BaseCF) base est | name: RF, estimator: RandomForestClassifier(ccp_alpha=0.1, class_weight='balanced', n_estimators=200,
oob_score=True, random_state=20)
(BaseCF) base est | name: KNNC, estimator: KNeighborsClassifier(n_neighbors=2, weights='distance')
(BaseCF) base est | name: GNB, estimator: GaussianNB()
(BaseCF) base est | name: QDA, estimator: QuadraticDiscriminantAnalysis()
(BaseCF) base est | name: MLPClassifier, estimator: MLPClassifier(alpha=1, max_iter=1000)
(BaseCF) Base predictors:
[1] RF: RandomForestClassifier(ccp_alpha=0.1, class_weight='balanced', n_estimators=200,
oob_score=True, random_state=20)
[2] QDA: QuadraticDiscriminantAnalysis()
[3] MLPClassifier: MLPClassifier(alpha=1, max_iter=1000)
[4] KNNC: KNeighborsClassifier(n_neighbors=2, weights='distance')
[5] GNB: GaussianNB()
[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 25.1s finished [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 0.2s finished [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 0.0s finished [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers. [Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 0.2s finished [Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.
[info] Saving X_meta (shape=(3750, 5)) at: /content/drive/MyDrive/Colab Notebooks/machine_learning_examples/cf_ensemble/data/train-0.npz
[Parallel(n_jobs=1)]: Done 5 out of 5 | elapsed: 27.4s finished
[info] Saving X_meta (shape=(1250, 5)) at: /content/drive/MyDrive/Colab Notebooks/machine_learning_examples/cf_ensemble/data/test-0.npz [info] Saving X_meta (shape=(1250, 5)) at: /content/drive/MyDrive/Colab Notebooks/machine_learning_examples/cf_ensemble/data/test-0.npz [result] 0.09859154929577464 (cf_write) Adding new attribute y: [0 0 0 0 0 ... 0 1 0 0 0] ... (cf_write) Saving X_meta at: /content/drive/MyDrive/Colab Notebooks/machine_learning_examples/cf_ensemble/data/test-0.npz
100%|██████████| 1/1 [01:10<00:00, 70.97s/it]
[info] list of base classifiers: ['RF' 'KNNC' 'GNB' 'QDA' 'MLPClassifier'] ================================================================================ R: Rating/probability matrix for the TRAIN set ================================================================================
Using stacking to make the first labeling prediction on test set¶
In [ ]:
Copied!
from collections import namedtuple
from sklearn.metrics import f1_score
alpha = 100
n_factors = 100
policy_threshold = 'fmax'
conf_measure = 'brier'
scoring_fn = f1_score
stacker = RandomForestClassifier(oob_score=True)
grid = uclf.hyperparameter_template(model='rf')
# A CF ensemble dataset consists of several parts: original (rating) matrix, re-estimated matrix, ...
# - namedtuple comes in handy
DataSet = namedtuple("DataSet", "X, Xh, L") # declare a `DataSet` type with the attributes: X, Xh and L
Hyperparams = namedtuple("Hyperparams", "alpha, n_factors, policy_threshold, conf_measure")
# The objects associated with traing split, hyperparameters are invariant across different prediction strategies
####################################################
meta = Hyperparams(policy_threshold=policy_threshold,
conf_measure=conf_measure,
alpha=alpha, n_factors=n_factors)
train_split = DataSet(X=R, Xh=None, L=L_train)
test_split = DataSet(X=T, Xh=None, L=None)
####################################################
# Instead of using majority vote ...
# lh = uc.estimateLabels(T, p_th=p_threshold) # We cannot use L_test (cheating), but we have to guesstimate
# ... Use another stacker
lh = lh_stacking = uc.estimateLabelsByStacking(stacker, grid, train_split, test_split, verbose=10)
perf_score = scoring_fn(L_test, lh)
acc_stacking = np.sum(lh == L_test) / (len(L_test)+0.0)
print(f'> [1] Labeling score ({scoring_fn.__name__}): {perf_score}')
print(f"> [1] Labeling accuracy: {acc_stacking}")
L = np.hstack((L_train, lh))
X = np.hstack((R, T))
assert len(U) == X.shape[0]
print(f"> shape(R):{R.shape} || shape(T): {T.shape} => shape(X): {X.shape}")
from collections import namedtuple
from sklearn.metrics import f1_score
alpha = 100
n_factors = 100
policy_threshold = 'fmax'
conf_measure = 'brier'
scoring_fn = f1_score
stacker = RandomForestClassifier(oob_score=True)
grid = uclf.hyperparameter_template(model='rf')
# A CF ensemble dataset consists of several parts: original (rating) matrix, re-estimated matrix, ...
# - namedtuple comes in handy
DataSet = namedtuple("DataSet", "X, Xh, L") # declare a `DataSet` type with the attributes: X, Xh and L
Hyperparams = namedtuple("Hyperparams", "alpha, n_factors, policy_threshold, conf_measure")
# The objects associated with traing split, hyperparameters are invariant across different prediction strategies
####################################################
meta = Hyperparams(policy_threshold=policy_threshold,
conf_measure=conf_measure,
alpha=alpha, n_factors=n_factors)
train_split = DataSet(X=R, Xh=None, L=L_train)
test_split = DataSet(X=T, Xh=None, L=None)
####################################################
# Instead of using majority vote ...
# lh = uc.estimateLabels(T, p_th=p_threshold) # We cannot use L_test (cheating), but we have to guesstimate
# ... Use another stacker
lh = lh_stacking = uc.estimateLabelsByStacking(stacker, grid, train_split, test_split, verbose=10)
perf_score = scoring_fn(L_test, lh)
acc_stacking = np.sum(lh == L_test) / (len(L_test)+0.0)
print(f'> [1] Labeling score ({scoring_fn.__name__}): {perf_score}')
print(f"> [1] Labeling accuracy: {acc_stacking}")
L = np.hstack((L_train, lh))
X = np.hstack((R, T))
assert len(U) == X.shape[0]
print(f"> shape(R):{R.shape} || shape(T): {T.shape} => shape(X): {X.shape}")
Fitting 10 folds for each of 24 candidates, totalling 240 fits
Best: 0.117860 using {'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.113676 (0.038609) with: {'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.113520 (0.038514) with: {'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
0.113564 (0.038664) with: {'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
0.113520 (0.038514) with: {'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
0.113408 (0.038568) with: {'bootstrap': True, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.113520 (0.038514) with: {'bootstrap': True, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
0.113408 (0.038568) with: {'bootstrap': True, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
0.113520 (0.038514) with: {'bootstrap': True, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
0.117860 (0.040120) with: {'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.116826 (0.041328) with: {'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
0.114701 (0.036436) with: {'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
0.112509 (0.038475) with: {'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
> shape(y_pred): (1250,)
[estimateLabelsByStacking] Best parameters:
{'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
> [1] Labeling score (f1_score): 0.18543046357615894
> [1] Labeling accuracy: 0.9016
> shape(R):(5, 3750) || shape(T): (5, 1250) => shape(X): (5, 5000)
In [ ]:
Copied!
def f_score(precision, recall, beta=1.0):
f_beta = (1 + beta**2) * (precision * recall) / ((beta**2 * precision) + recall)
return f_beta
# from common import f_score
# workflow: p_threshold -> lh (est. labels) -> color matrix
Pc_stacking, Lh0 = pmodel.color_matrix(T, lh_stacking, p_threshold) # Mc: Color matrix evaluated via estimated labels
Pf_stacking = pmodel.to_preference(Pc_stacking, neutral=0.0)
# => {TP, TN}-entries are desirable and thus encoded as 1s in `Pf_stacking` whereas {FP, FN}-entries are not desirable and encoded as 0s
metrics = pmodel.eval_estimated_probability_filter(Pf_stacking, T, L_test, p_threshold, eps=1e-3)
highlight("Guesstimated labeling (on T) via STACKING")
print(f"> Labeling accuracy: {acc_stacking}")
print(f"> Reliable-to-correct ratio: {metrics['p_overlap']}") # Fraction of entries predicted reliable and are actually correct (TPs or TNs)
print(f"> Precision: {metrics['precision']}, Recall: {metrics['recall']}")
print(f"> Predcitio(TP): {metrics['precision_tp']}, Recall(TP): {metrics['recall_tp']} => f1(TP): {f_score(metrics['precision_tp'], metrics['recall_tp'])}")
print(f"> Error rate: {metrics['p_missed']}") # Probability of predicting reliable but hitting either FPs or FNs
# How does it fair with majority vote?
###############################################
# labeling by majority vote
lh_max_vote = uc.estimateLabels(T, p_th=p_threshold, pos_label=1)
acc_max_vote = np.sum(lh_max_vote == L_test) / (len(L_test)+0.0)
Pc_maxvote, Lh0 = pmodel.color_matrix(T, lh_max_vote, p_threshold) # Mc: Color matrix evaluated via estimated labels
Pf_maxvote = pmodel.to_preference(Pc_maxvote, neutral=0.0)
# => {TP, TN}-entries are desirable and thus encoded as 1s in `Pf_maxvote` whereas {FP, FN}-entries are not desirable hence encoded as 0s
metrics = pmodel.eval_estimated_probability_filter(Pf_maxvote, T, L_test, p_threshold, eps=1e-3)
highlight("Guesstimated labeling (on T) via MAJORITY VOTE")
print(f"> Labeling accuracy: {acc_max_vote}")
print(f"> Reliable-to-correct ratio: {metrics['p_overlap']}") # Fraction of entries predicted reliable and are actually correct (TPs or TNs)
print(f"> Precision: {metrics['precision']}, Recall: {metrics['recall']}")
print(f"> Predcitio(TP): {metrics['precision_tp']}, Recall(TP): {metrics['recall_tp']} => f1(TP): {f_score(metrics['precision_tp'], metrics['recall_tp'])}")
print(f"> Error rate: {metrics['p_missed']}") # Probability of predicting reliable but hitting either FPs or FNs
def f_score(precision, recall, beta=1.0):
f_beta = (1 + beta**2) * (precision * recall) / ((beta**2 * precision) + recall)
return f_beta
# from common import f_score
# workflow: p_threshold -> lh (est. labels) -> color matrix
Pc_stacking, Lh0 = pmodel.color_matrix(T, lh_stacking, p_threshold) # Mc: Color matrix evaluated via estimated labels
Pf_stacking = pmodel.to_preference(Pc_stacking, neutral=0.0)
# => {TP, TN}-entries are desirable and thus encoded as 1s in `Pf_stacking` whereas {FP, FN}-entries are not desirable and encoded as 0s
metrics = pmodel.eval_estimated_probability_filter(Pf_stacking, T, L_test, p_threshold, eps=1e-3)
highlight("Guesstimated labeling (on T) via STACKING")
print(f"> Labeling accuracy: {acc_stacking}")
print(f"> Reliable-to-correct ratio: {metrics['p_overlap']}") # Fraction of entries predicted reliable and are actually correct (TPs or TNs)
print(f"> Precision: {metrics['precision']}, Recall: {metrics['recall']}")
print(f"> Predcitio(TP): {metrics['precision_tp']}, Recall(TP): {metrics['recall_tp']} => f1(TP): {f_score(metrics['precision_tp'], metrics['recall_tp'])}")
print(f"> Error rate: {metrics['p_missed']}") # Probability of predicting reliable but hitting either FPs or FNs
# How does it fair with majority vote?
###############################################
# labeling by majority vote
lh_max_vote = uc.estimateLabels(T, p_th=p_threshold, pos_label=1)
acc_max_vote = np.sum(lh_max_vote == L_test) / (len(L_test)+0.0)
Pc_maxvote, Lh0 = pmodel.color_matrix(T, lh_max_vote, p_threshold) # Mc: Color matrix evaluated via estimated labels
Pf_maxvote = pmodel.to_preference(Pc_maxvote, neutral=0.0)
# => {TP, TN}-entries are desirable and thus encoded as 1s in `Pf_maxvote` whereas {FP, FN}-entries are not desirable hence encoded as 0s
metrics = pmodel.eval_estimated_probability_filter(Pf_maxvote, T, L_test, p_threshold, eps=1e-3)
highlight("Guesstimated labeling (on T) via MAJORITY VOTE")
print(f"> Labeling accuracy: {acc_max_vote}")
print(f"> Reliable-to-correct ratio: {metrics['p_overlap']}") # Fraction of entries predicted reliable and are actually correct (TPs or TNs)
print(f"> Precision: {metrics['precision']}, Recall: {metrics['recall']}")
print(f"> Predcitio(TP): {metrics['precision_tp']}, Recall(TP): {metrics['recall_tp']} => f1(TP): {f_score(metrics['precision_tp'], metrics['recall_tp'])}")
print(f"> Error rate: {metrics['p_missed']}") # Probability of predicting reliable but hitting either FPs or FNs
================================================================================ Guesstimated labeling (on T) via STACKING ================================================================================ > Labeling accuracy: 0.9016 > Reliable-to-correct ratio: 0.9016 > Precision: 0.903357871881391, Recall: 0.8718037912121746 > Predcitio(TP): 0.0208955145912259, Recall(TP): 0.13827126352774438 => f1(TP): 0.03630467662581742 > Error rate: 3.6060342882118244e-05 ================================================================================ Guesstimated labeling (on T) via MAJORITY VOTE ================================================================================ > Labeling accuracy: 0.3512 > Reliable-to-correct ratio: 0.3512 > Precision: 0.34072773162319747, Recall: 0.4922576549306248 > Predcitio(TP): 0.08150546323393239, Recall(TP): 0.807405413813793 => f1(TP): 0.14806422999129273 > Error rate: 0.00016432498873370703
Training CFNet¶
In [ ]:
Copied!
import cf_models as cm
n_users, n_items = X.shape
fold_number = 0
test_size = 0.1
policy_threshold = 'fmax'
conf_measure = 'brier'
n_factors = 100
alpha = 100
lr = 0.001
batch_size = 64
epochs = 200
# Note: confidence_weighted_loss converges relatively faster compared to MSE, BCE etc
loss_fn = tf.keras.losses.BinaryCrossentropy() # Options: cm.confidence_weighted_loss, cm.c_squared_loss, tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.MeanSquaredError(), ...
cf_model = cm.get_cfnet_compiled(n_users, n_items, n_factors, loss=loss_fn, lr=lr)
# cf_model = cm.get_cfnet_approximating_labels(n_users, n_items, n_factors)
# Configure `target_type` (Options: 'generic', 'rating', 'label')
# 1. Choose 'label' if the BCE loss is used (because the CF model in this case attempts to approximates the label encoded in 0 and 1)
# 2. Choose 'rating' if MSE is used (because the CF model in this case approximates the rating, which is a regression problem)
# 3. Choose 'generic' for customized loss function with potentially more complex labeling information where "y_true" is a matrix
#
# Note that you are unlikely need to configure `target_type` because cf_models module has a method that will determine this for you automatically
# target_type = 'label' # Options: 'generic', 'rating', 'label'
# Configure `conf_type` with options: 'Cn', 'Cw', 'C0', or your customized weights
conf_type = 'Cn'
cf_model = cm.training_pipeline(input_model=(cf_model, loss_fn),
input_data=(R, T, U, L_train, ),
lh = lh_stacking, # supply the pre-computed estimated labels for T
# Should we combine R and T into a single matrix X? Set to True if so
is_cascade = True, # Set to True here to combine R and T into X
# SGD optimization parameters
test_size = test_size,
epochs = epochs,
batch_size=batch_size,
# CF hyperparameters
# n_factors=n_factors, # this is factored into model definition
alpha=alpha,
conf_measure=conf_measure,
policy_threshold=policy_threshold,
conf_type=conf_type, # default sparse confidence matrix (Cn)
# target_type=target_type,
fold_number=fold_number)
import cf_models as cm
n_users, n_items = X.shape
fold_number = 0
test_size = 0.1
policy_threshold = 'fmax'
conf_measure = 'brier'
n_factors = 100
alpha = 100
lr = 0.001
batch_size = 64
epochs = 200
# Note: confidence_weighted_loss converges relatively faster compared to MSE, BCE etc
loss_fn = tf.keras.losses.BinaryCrossentropy() # Options: cm.confidence_weighted_loss, cm.c_squared_loss, tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.MeanSquaredError(), ...
cf_model = cm.get_cfnet_compiled(n_users, n_items, n_factors, loss=loss_fn, lr=lr)
# cf_model = cm.get_cfnet_approximating_labels(n_users, n_items, n_factors)
# Configure `target_type` (Options: 'generic', 'rating', 'label')
# 1. Choose 'label' if the BCE loss is used (because the CF model in this case attempts to approximates the label encoded in 0 and 1)
# 2. Choose 'rating' if MSE is used (because the CF model in this case approximates the rating, which is a regression problem)
# 3. Choose 'generic' for customized loss function with potentially more complex labeling information where "y_true" is a matrix
#
# Note that you are unlikely need to configure `target_type` because cf_models module has a method that will determine this for you automatically
# target_type = 'label' # Options: 'generic', 'rating', 'label'
# Configure `conf_type` with options: 'Cn', 'Cw', 'C0', or your customized weights
conf_type = 'Cn'
cf_model = cm.training_pipeline(input_model=(cf_model, loss_fn),
input_data=(R, T, U, L_train, ),
lh = lh_stacking, # supply the pre-computed estimated labels for T
# Should we combine R and T into a single matrix X? Set to True if so
is_cascade = True, # Set to True here to combine R and T into X
# SGD optimization parameters
test_size = test_size,
epochs = epochs,
batch_size=batch_size,
# CF hyperparameters
# n_factors=n_factors, # this is factored into model definition
alpha=alpha,
conf_measure=conf_measure,
policy_threshold=policy_threshold,
conf_type=conf_type, # default sparse confidence matrix (Cn)
# target_type=target_type,
fold_number=fold_number)
[merge] Merging 'L_train' and 'lh': len(L_train): 3750 || len(lh): 1250 => len(L): 5000 [merge] Merging 'R' and 'T': shape(R):(5, 3750) || shape(T): (5, 1250) => shape(X): (5, 5000) (make_cn) Using WEIGHTED confidence matrix to approximate ratings ... [info] Confidence matrix type: Cn, target data type: label Epoch 1/200 352/352 [==============================] - 3s 7ms/step - loss: 2.7584 - val_loss: 2.4234 Epoch 2/200 352/352 [==============================] - 2s 5ms/step - loss: 2.0907 - val_loss: 1.6369 Epoch 3/200 352/352 [==============================] - 2s 5ms/step - loss: 1.8599 - val_loss: 1.5557 Epoch 4/200 352/352 [==============================] - 2s 5ms/step - loss: 0.9408 - val_loss: 0.9729 Epoch 5/200 352/352 [==============================] - 2s 5ms/step - loss: 0.6643 - val_loss: 0.8774 Epoch 6/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5643 - val_loss: 0.8054 Epoch 7/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5195 - val_loss: 0.7810 Epoch 8/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4920 - val_loss: 0.7680 Epoch 9/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4763 - val_loss: 0.7497 Epoch 10/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4626 - val_loss: 0.7386 Epoch 11/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4517 - val_loss: 0.7083 Epoch 12/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4393 - val_loss: 0.7055 Epoch 13/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4242 - val_loss: 0.6947 Epoch 14/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4087 - val_loss: 0.6709 Epoch 15/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3927 - val_loss: 0.6453 Epoch 16/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3783 - val_loss: 0.6286 Epoch 17/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3622 - val_loss: 0.6171 Epoch 18/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3487 - val_loss: 0.6012 Epoch 19/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3352 - val_loss: 0.5912 Epoch 20/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3238 - val_loss: 0.5832 Epoch 21/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3148 - val_loss: 0.5769 Epoch 22/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3072 - val_loss: 0.5706 Epoch 23/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3006 - val_loss: 0.5645 Epoch 24/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2945 - val_loss: 0.5575 Epoch 25/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2887 - val_loss: 0.5506 Epoch 26/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2832 - val_loss: 0.5432 Epoch 27/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2779 - val_loss: 0.5351 Epoch 28/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2727 - val_loss: 0.5275 Epoch 29/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2676 - val_loss: 0.5196 Epoch 30/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2627 - val_loss: 0.5116 Epoch 31/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2578 - val_loss: 0.5039 Epoch 32/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2531 - val_loss: 0.4960 Epoch 33/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2484 - val_loss: 0.4875 Epoch 34/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2438 - val_loss: 0.4800 Epoch 35/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2393 - val_loss: 0.4723 Epoch 36/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2349 - val_loss: 0.4646 Epoch 37/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2307 - val_loss: 0.4576 Epoch 38/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2265 - val_loss: 0.4496 Epoch 39/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2225 - val_loss: 0.4424 Epoch 40/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2186 - val_loss: 0.4351 Epoch 41/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2144 - val_loss: 0.4283 Epoch 42/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2106 - val_loss: 0.4214 Epoch 43/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2067 - val_loss: 0.4143 Epoch 44/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2026 - val_loss: 0.4072 Epoch 45/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1995 - val_loss: 0.4000 Epoch 46/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1962 - val_loss: 0.3943 Epoch 47/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1915 - val_loss: 0.3865 Epoch 48/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1878 - val_loss: 0.3806 Epoch 49/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1841 - val_loss: 0.3741 Epoch 50/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1809 - val_loss: 0.3680 Epoch 51/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1783 - val_loss: 0.3619 Epoch 52/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1740 - val_loss: 0.3557 Epoch 53/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1708 - val_loss: 0.3493 Epoch 54/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1680 - val_loss: 0.3438 Epoch 55/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1640 - val_loss: 0.3373 Epoch 56/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1610 - val_loss: 0.3320 Epoch 57/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1587 - val_loss: 0.3283 Epoch 58/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1552 - val_loss: 0.3214 Epoch 59/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1518 - val_loss: 0.3150 Epoch 60/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1489 - val_loss: 0.3101 Epoch 61/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1460 - val_loss: 0.3052 Epoch 62/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1438 - val_loss: 0.3000 Epoch 63/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1405 - val_loss: 0.2956 Epoch 64/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1378 - val_loss: 0.2898 Epoch 65/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1350 - val_loss: 0.2845 Epoch 66/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1325 - val_loss: 0.2800 Epoch 67/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1298 - val_loss: 0.2749 Epoch 68/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1272 - val_loss: 0.2711 Epoch 69/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1251 - val_loss: 0.2664 Epoch 70/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1224 - val_loss: 0.2613 Epoch 71/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1197 - val_loss: 0.2568 Epoch 72/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1173 - val_loss: 0.2525 Epoch 73/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1151 - val_loss: 0.2481 Epoch 74/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1128 - val_loss: 0.2443 Epoch 75/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1107 - val_loss: 0.2402 Epoch 76/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1085 - val_loss: 0.2360 Epoch 77/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1063 - val_loss: 0.2317 Epoch 78/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1040 - val_loss: 0.2277 Epoch 79/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1019 - val_loss: 0.2240 Epoch 80/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1003 - val_loss: 0.2205 Epoch 81/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0979 - val_loss: 0.2163 Epoch 82/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0959 - val_loss: 0.2130 Epoch 83/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0940 - val_loss: 0.2090 Epoch 84/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0922 - val_loss: 0.2058 Epoch 85/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0904 - val_loss: 0.2022 Epoch 86/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0884 - val_loss: 0.1989 Epoch 87/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0868 - val_loss: 0.1953 Epoch 88/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0848 - val_loss: 0.1922 Epoch 89/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0831 - val_loss: 0.1888 Epoch 90/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0815 - val_loss: 0.1855 Epoch 91/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0799 - val_loss: 0.1826 Epoch 92/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0782 - val_loss: 0.1791 Epoch 93/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0766 - val_loss: 0.1760 Epoch 94/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0750 - val_loss: 0.1732 Epoch 95/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0736 - val_loss: 0.1706 Epoch 96/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0720 - val_loss: 0.1673 Epoch 97/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0705 - val_loss: 0.1643 Epoch 98/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0690 - val_loss: 0.1616 Epoch 99/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0677 - val_loss: 0.1590 Epoch 100/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0663 - val_loss: 0.1564 Epoch 101/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0649 - val_loss: 0.1537 Epoch 102/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0635 - val_loss: 0.1510 Epoch 103/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0623 - val_loss: 0.1486 Epoch 104/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0610 - val_loss: 0.1460 Epoch 105/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0597 - val_loss: 0.1436 Epoch 106/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0585 - val_loss: 0.1414 Epoch 107/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0573 - val_loss: 0.1388 Epoch 108/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0561 - val_loss: 0.1363 Epoch 109/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0550 - val_loss: 0.1343 Epoch 110/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0538 - val_loss: 0.1319 Epoch 111/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0527 - val_loss: 0.1295 Epoch 112/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0516 - val_loss: 0.1275 Epoch 113/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0505 - val_loss: 0.1254 Epoch 114/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0495 - val_loss: 0.1234 Epoch 115/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0485 - val_loss: 0.1213 Epoch 116/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0475 - val_loss: 0.1193 Epoch 117/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0465 - val_loss: 0.1173 Epoch 118/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0455 - val_loss: 0.1154 Epoch 119/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0446 - val_loss: 0.1134 Epoch 120/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0437 - val_loss: 0.1115 Epoch 121/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0427 - val_loss: 0.1098 Epoch 122/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0418 - val_loss: 0.1079 Epoch 123/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0410 - val_loss: 0.1062 Epoch 124/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0401 - val_loss: 0.1043 Epoch 125/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0393 - val_loss: 0.1026 Epoch 126/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0385 - val_loss: 0.1010 Epoch 127/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0377 - val_loss: 0.0995 Epoch 128/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0369 - val_loss: 0.0977 Epoch 129/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0361 - val_loss: 0.0961 Epoch 130/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0354 - val_loss: 0.0946 Epoch 131/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0346 - val_loss: 0.0930 Epoch 132/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0339 - val_loss: 0.0915 Epoch 133/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0332 - val_loss: 0.0900 Epoch 134/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0325 - val_loss: 0.0886 Epoch 135/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0318 - val_loss: 0.0872 Epoch 136/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0312 - val_loss: 0.0858 Epoch 137/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0305 - val_loss: 0.0845 Epoch 138/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0299 - val_loss: 0.0831 Epoch 139/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0292 - val_loss: 0.0818 Epoch 140/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0286 - val_loss: 0.0805 Epoch 141/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0280 - val_loss: 0.0792 Epoch 142/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0275 - val_loss: 0.0779 Epoch 143/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0269 - val_loss: 0.0767 Epoch 144/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0263 - val_loss: 0.0756 Epoch 145/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0258 - val_loss: 0.0744 Epoch 146/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0252 - val_loss: 0.0732 Epoch 147/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0247 - val_loss: 0.0721 Epoch 148/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0242 - val_loss: 0.0710 Epoch 149/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0237 - val_loss: 0.0699 Epoch 150/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0232 - val_loss: 0.0688 Epoch 151/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0227 - val_loss: 0.0677 Epoch 152/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0222 - val_loss: 0.0666 Epoch 153/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0218 - val_loss: 0.0657 Epoch 154/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0213 - val_loss: 0.0647 Epoch 155/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0209 - val_loss: 0.0638 Epoch 156/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0204 - val_loss: 0.0628 Epoch 157/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0200 - val_loss: 0.0618 Epoch 158/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0196 - val_loss: 0.0609 Epoch 159/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0192 - val_loss: 0.0600 Epoch 160/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0188 - val_loss: 0.0591 Epoch 161/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0184 - val_loss: 0.0582 Epoch 162/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0180 - val_loss: 0.0574 Epoch 163/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0176 - val_loss: 0.0565 Epoch 164/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0172 - val_loss: 0.0558 Epoch 165/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0169 - val_loss: 0.0549 Epoch 166/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0165 - val_loss: 0.0541 Epoch 167/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0162 - val_loss: 0.0533 Epoch 168/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0158 - val_loss: 0.0526 Epoch 169/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0155 - val_loss: 0.0518 Epoch 170/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0152 - val_loss: 0.0510 Epoch 171/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0149 - val_loss: 0.0503 Epoch 172/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0146 - val_loss: 0.0495 Epoch 173/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0143 - val_loss: 0.0489 Epoch 174/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0140 - val_loss: 0.0481 Epoch 175/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0137 - val_loss: 0.0475 Epoch 176/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0134 - val_loss: 0.0469 Epoch 177/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0131 - val_loss: 0.0462 Epoch 178/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0128 - val_loss: 0.0455 Epoch 179/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0126 - val_loss: 0.0448 Epoch 180/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0123 - val_loss: 0.0442 Epoch 181/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0120 - val_loss: 0.0437 Epoch 182/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0118 - val_loss: 0.0430 Epoch 183/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0115 - val_loss: 0.0424 Epoch 184/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0113 - val_loss: 0.0418 Epoch 185/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0111 - val_loss: 0.0413 Epoch 186/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0108 - val_loss: 0.0408 Epoch 187/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0106 - val_loss: 0.0403 Epoch 188/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0104 - val_loss: 0.0397 Epoch 189/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0102 - val_loss: 0.0391 Epoch 190/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0100 - val_loss: 0.0386 Epoch 191/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0098 - val_loss: 0.0381 Epoch 192/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0095 - val_loss: 0.0377 Epoch 193/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0094 - val_loss: 0.0371 Epoch 194/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0092 - val_loss: 0.0366 Epoch 195/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0090 - val_loss: 0.0362 Epoch 196/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0088 - val_loss: 0.0358 Epoch 197/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0086 - val_loss: 0.0353 Epoch 198/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0084 - val_loss: 0.0349 Epoch 199/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0082 - val_loss: 0.0344 Epoch 200/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0081 - val_loss: 0.0340
In [ ]:
Copied!
Pc, _ = pmodel.color_matrix(X, L, p_th=p_threshold)
y_colors = pmodel.verify_colors(Pc)
Pc, _ = pmodel.color_matrix(X, L, p_th=p_threshold)
y_colors = pmodel.verify_colors(Pc)
In [ ]:
Copied!
analyzer = cm.analyze_reconstruction(cf_model,
X=(R, T),
L=(L_train, lh_stacking),
Pc=Pc, p_threshold=p_threshold, policy_threshold=policy_threshold)
highlight("Reestimate the entire rating matrix (X) with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=False)
highlight("Reestimate ONLY the unreliable entries in X with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=True, verbose=2)
# Returned value `reestimated` is a dictionary with keys: `Rh`, `Th`, `ratings`, `p_threshold`
analyzer = cm.analyze_reconstruction(cf_model,
X=(R, T),
L=(L_train, lh_stacking),
Pc=Pc, p_threshold=p_threshold, policy_threshold=policy_threshold)
highlight("Reestimate the entire rating matrix (X) with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=False)
highlight("Reestimate ONLY the unreliable entries in X with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=True, verbose=2)
# Returned value `reestimated` is a dictionary with keys: `Rh`, `Th`, `ratings`, `p_threshold`
================================================================================ Reestimate the entire rating matrix (X) with learned latent factors/embeddings ================================================================================ [info] From R to Rh, delta(Frobenius norm)= 74.88749139328851 [info] From T to Th, delta(Frobenius norm)= 38.692016460740206 [info] How different are lh and lh_new? 0.6816 [result] Majority vote: F1 score with the original T: 0.18981018981018982 [result] Majority vote: F1 score with re-estimated Th using original p_threshold: 0.19364161849710984 [result] Majority vote: F1 score with re-estimated Th: 0.1879194630872483 [result] Stacking: F1 score with the original T: 0.11188811188811189 [result] Stacking: F1 score with re-estimated Th: 0.18543046357615894 [result] Best settings (complete): lh2_maxvote_pth_unadjusted, score: 0.19364161849710984 ================================================================================ Reestimate ONLY the unreliable entries in X with learned latent factors/embeddings ================================================================================ [info] From R to Rh, delta(Frobenius norm)= 68.28001141208821 [info] From T to Th, delta(Frobenius norm)= 34.91879838477874 [info] How different are lh and lh_new? 0.6832 [result] Majority vote: F1 score with the original T: 0.18981018981018982 [result] Majority vote: F1 score with re-estimated Th using original p_threshold: 0.199616122840691 [result] Majority vote: F1 score with re-estimated Th: 0.18543046357615894 [result] Stacking: F1 score with the original T: 0.11188811188811189 [result] Stacking: F1 score with re-estimated Th: 0.18543046357615894 [result] Methods ranked: [(0.199616122840691, 'lh2_maxvote_pth_unadjusted'), (0.18981018981018982, 'lh_maxvote'), (0.18543046357615894, 'lh2_maxvote_pth_adjusted')] [result] Best settings (unreliable only): lh2_maxvote_pth_unadjusted, score: 0.199616122840691 [help] Reestiamted quantities are available through the following keys: - ratings - p_threshold2 - lh_maxvote - lh2_maxvote_pth_unadjusted - lh2_maxvote_pth_adjusted - score_lh_maxvote - score_baseline - score_lh2_maxvote_pth_unadjusted - score_lh2_maxvote_pth_adjusted - score_lh_stacker - score_lh2_stacker_pth_adjusted - best_params - best_params_score
What if we tuned probability matrices even further using CFNet?¶
In [ ]:
Copied!
# A. Reestimate entire matrix
# Rh, Th = reestimate(model, X, n_train=n_train)
# B. Reestimate only unreliable entries (better)
# Rh, Th = reestimate_unreliable_only(model, X, Pc=Pc, n_train=n_train)
reestimated = analyzer(L_test, unreliable_only=True)
R2, T2 = reestimated['ratings'] #
p_threshold2 = reestimated['p_threshold2']
# ... Use another stacker
stacker = RandomForestClassifier(oob_score=True)
grid = uclf.hyperparameter_template(model='rf')
# New `R` and `T` but the labels, of course, remain the same
train_split = DataSet(X=R2, Xh=None, L=L_train)
test_split = DataSet(X=T2, Xh=None, L=None)
lh2 = lh_stacking = uc.estimateLabelsByStacking(stacker, grid, train_split, test_split, verbose=5)
perf_score = scoring_fn(L_test, lh2)
acc_stacking = np.sum(lh2 == L_test) / (len(L_test)+0.0)
print(f'> [2] Labeling score ({scoring_fn.__name__}): {perf_score}')
print(f"> [2] Labeling accuracy: {acc_stacking}")
L2 = np.hstack((L_train, lh2))
X2 = np.hstack((R2, T2))
# A. Reestimate entire matrix
# Rh, Th = reestimate(model, X, n_train=n_train)
# B. Reestimate only unreliable entries (better)
# Rh, Th = reestimate_unreliable_only(model, X, Pc=Pc, n_train=n_train)
reestimated = analyzer(L_test, unreliable_only=True)
R2, T2 = reestimated['ratings'] #
p_threshold2 = reestimated['p_threshold2']
# ... Use another stacker
stacker = RandomForestClassifier(oob_score=True)
grid = uclf.hyperparameter_template(model='rf')
# New `R` and `T` but the labels, of course, remain the same
train_split = DataSet(X=R2, Xh=None, L=L_train)
test_split = DataSet(X=T2, Xh=None, L=None)
lh2 = lh_stacking = uc.estimateLabelsByStacking(stacker, grid, train_split, test_split, verbose=5)
perf_score = scoring_fn(L_test, lh2)
acc_stacking = np.sum(lh2 == L_test) / (len(L_test)+0.0)
print(f'> [2] Labeling score ({scoring_fn.__name__}): {perf_score}')
print(f"> [2] Labeling accuracy: {acc_stacking}")
L2 = np.hstack((L_train, lh2))
X2 = np.hstack((R2, T2))
[info] From R to Rh, delta(Frobenius norm)= 68.28001141208821
[info] From T to Th, delta(Frobenius norm)= 34.91879838477874
[info] How different are lh and lh_new? 0.6832
[result] Majority vote: F1 score with the original T: 0.18981018981018982
[result] Majority vote: F1 score with re-estimated Th using original p_threshold: 0.199616122840691
[result] Majority vote: F1 score with re-estimated Th: 0.18543046357615894
[result] Stacking: F1 score with the original T: 0.11188811188811189
[result] Stacking: F1 score with re-estimated Th: 0.18543046357615894
[result] Best settings (unreliable only): lh2_maxvote_pth_unadjusted, score: 0.199616122840691
Fitting 10 folds for each of 24 candidates, totalling 240 fits
Best: 1.000000 using {'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
1.000000 (0.000000) with: {'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': 10, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 300}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 200}
0.000000 (0.000000) with: {'bootstrap': False, 'max_depth': None, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 4, 'n_estimators': 300}
> shape(y_pred): (1250,)
[estimateLabelsByStacking] Best parameters:
{'bootstrap': True, 'max_depth': 8, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}
> [2] Labeling score (f1_score): 0.18543046357615894
> [2] Labeling accuracy: 0.9016
In [ ]:
Copied!
from common import f_score
# workflow: p_threshold -> lh -> color matrix
Pc_stacking2, Lh2 = pmodel.color_matrix(T2, lh2, p_threshold2) # Mc: Color matrix evaluated via estimated labels
Pf_stacking2 = pmodel.to_preference(Pc_stacking2, neutral=0.0)
# => {TP, TN}-entries are desirable and thus encoded as 1s in `Tpf` whereas {FP, FN}-entries are not desirable and encoded as 0s
metrics = pmodel.eval_estimated_probability_filter(Pf_stacking2, T2, L_test, p_threshold2, eps=1e-3)
highlight("Guesstimated labeling (on T2) via STACKING")
print(f"> Labeling accuracy: {acc_stacking}")
print(f"> Reliable-to-correct ratio: {metrics['p_overlap']}") # Fraction of entries predicted reliable and are actually correct (TPs or TNs)
print(f"> Precision: {metrics['precision']}, Recall: {metrics['recall']}")
print(f"> Precision(TP): {metrics['precision_tp']}, Recall(TP): {metrics['recall_tp']} => f1(TP): {f_score(metrics['precision_tp'], metrics['recall_tp'])}")
# NOTE: Precision(TP) is defined as P(TP|reliable), i.e. probability of being TP given predicted "reliable" (in an average sense)
print(f"> Error rate: {metrics['p_missed']}") # Probability of predicting reliable but hitting either FPs or FNs
from common import f_score
# workflow: p_threshold -> lh -> color matrix
Pc_stacking2, Lh2 = pmodel.color_matrix(T2, lh2, p_threshold2) # Mc: Color matrix evaluated via estimated labels
Pf_stacking2 = pmodel.to_preference(Pc_stacking2, neutral=0.0)
# => {TP, TN}-entries are desirable and thus encoded as 1s in `Tpf` whereas {FP, FN}-entries are not desirable and encoded as 0s
metrics = pmodel.eval_estimated_probability_filter(Pf_stacking2, T2, L_test, p_threshold2, eps=1e-3)
highlight("Guesstimated labeling (on T2) via STACKING")
print(f"> Labeling accuracy: {acc_stacking}")
print(f"> Reliable-to-correct ratio: {metrics['p_overlap']}") # Fraction of entries predicted reliable and are actually correct (TPs or TNs)
print(f"> Precision: {metrics['precision']}, Recall: {metrics['recall']}")
print(f"> Precision(TP): {metrics['precision_tp']}, Recall(TP): {metrics['recall_tp']} => f1(TP): {f_score(metrics['precision_tp'], metrics['recall_tp'])}")
# NOTE: Precision(TP) is defined as P(TP|reliable), i.e. probability of being TP given predicted "reliable" (in an average sense)
print(f"> Error rate: {metrics['p_missed']}") # Probability of predicting reliable but hitting either FPs or FNs
================================================================================ Guesstimated labeling (on T2) via STACKING ================================================================================ > Labeling accuracy: 0.9016 > Reliable-to-correct ratio: 0.9016 > Precision: 0.9013352797816863, Recall: 0.9741488638197191 > Precision(TP): 0.013952558510552418, Recall(TP): 0.36841911358361273 => f1(TP): 0.02688687271485588 > Error rate: 1.966603572453927e-05
Applying the same training pipeline given earlier using new rating matrices ...¶
In [ ]:
Copied!
n_users, n_items = X.shape
# Uncomment previously-defined parameters below if needed
# fold_number = 0
# test_size = 0.1
# policy_threshold = 'fmax'
# conf_measure = 'brier'
# n_factors = 100
# alpha = 100
# lr = 0.001
# batch_size = 64
# epochs = 60 # Note: confidence_weighted_loss converges relatively faster compared to MSE, BCE etc.
# target_type = 'generic' # Options: 'generic', 'rating', 'label'
# conf_type = 'Cn' # Options: 'Cn', 'Cw', 'C0', or your customized weights
loss_fn = tf.keras.losses.BinaryCrossentropy() # Options: cm.confidence_weighted_loss, cm.c_squared_loss, tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.MeanSquaredError(), ...
cf_model = cm.get_cfnet_compiled(n_users, n_items, n_factors, loss=loss_fn, lr=lr)
cf_model = cm.training_pipeline(input_model=(cf_model, loss_fn),
input_data=(R2, T2, U, L_train, ),
lh = lh2, # supply the pre-computed estimated labels for T
# Should we combine R and T into a single matrix X? Set to True if so
is_cascade = True, # Set to True here to combine R and T into X
# SGD optimization parameters
test_size = test_size,
epochs = epochs,
batch_size=batch_size,
# CF hyperparameters
# n_factors=n_factors, # this is factored into model definition
alpha=alpha,
conf_measure=conf_measure,
policy_threshold=policy_threshold,
conf_type=conf_type, # default sparse confidence matrix (Cn)
# target_type=target_type,
fold_number=fold_number)
n_users, n_items = X.shape
# Uncomment previously-defined parameters below if needed
# fold_number = 0
# test_size = 0.1
# policy_threshold = 'fmax'
# conf_measure = 'brier'
# n_factors = 100
# alpha = 100
# lr = 0.001
# batch_size = 64
# epochs = 60 # Note: confidence_weighted_loss converges relatively faster compared to MSE, BCE etc.
# target_type = 'generic' # Options: 'generic', 'rating', 'label'
# conf_type = 'Cn' # Options: 'Cn', 'Cw', 'C0', or your customized weights
loss_fn = tf.keras.losses.BinaryCrossentropy() # Options: cm.confidence_weighted_loss, cm.c_squared_loss, tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.MeanSquaredError(), ...
cf_model = cm.get_cfnet_compiled(n_users, n_items, n_factors, loss=loss_fn, lr=lr)
cf_model = cm.training_pipeline(input_model=(cf_model, loss_fn),
input_data=(R2, T2, U, L_train, ),
lh = lh2, # supply the pre-computed estimated labels for T
# Should we combine R and T into a single matrix X? Set to True if so
is_cascade = True, # Set to True here to combine R and T into X
# SGD optimization parameters
test_size = test_size,
epochs = epochs,
batch_size=batch_size,
# CF hyperparameters
# n_factors=n_factors, # this is factored into model definition
alpha=alpha,
conf_measure=conf_measure,
policy_threshold=policy_threshold,
conf_type=conf_type, # default sparse confidence matrix (Cn)
# target_type=target_type,
fold_number=fold_number)
[merge] Merging 'L_train' and 'lh': len(L_train): 3750 || len(lh): 1250 => len(L): 5000 [merge] Merging 'R' and 'T': shape(R):(5, 3750) || shape(T): (5, 1250) => shape(X): (5, 5000) (make_cn) Using WEIGHTED confidence matrix to approximate ratings ... [info] Confidence matrix type: Cn, target data type: label Epoch 1/200 352/352 [==============================] - 3s 7ms/step - loss: 2.9438 - val_loss: 2.6048 Epoch 2/200 352/352 [==============================] - 2s 5ms/step - loss: 2.7232 - val_loss: 2.2409 Epoch 3/200 352/352 [==============================] - 2s 5ms/step - loss: 2.1574 - val_loss: 1.6957 Epoch 4/200 352/352 [==============================] - 2s 5ms/step - loss: 1.5828 - val_loss: 1.3034 Epoch 5/200 352/352 [==============================] - 2s 5ms/step - loss: 1.2473 - val_loss: 1.0845 Epoch 6/200 352/352 [==============================] - 2s 5ms/step - loss: 1.0837 - val_loss: 0.9817 Epoch 7/200 352/352 [==============================] - 2s 5ms/step - loss: 1.0070 - val_loss: 0.9230 Epoch 8/200 352/352 [==============================] - 2s 5ms/step - loss: 0.9587 - val_loss: 0.8876 Epoch 9/200 352/352 [==============================] - 2s 5ms/step - loss: 0.9257 - val_loss: 0.8608 Epoch 10/200 352/352 [==============================] - 2s 5ms/step - loss: 0.8996 - val_loss: 0.8391 Epoch 11/200 352/352 [==============================] - 2s 5ms/step - loss: 0.8768 - val_loss: 0.8186 Epoch 12/200 352/352 [==============================] - 2s 5ms/step - loss: 0.8559 - val_loss: 0.8004 Epoch 13/200 352/352 [==============================] - 2s 5ms/step - loss: 0.8358 - val_loss: 0.7834 Epoch 14/200 352/352 [==============================] - 2s 5ms/step - loss: 0.8163 - val_loss: 0.7663 Epoch 15/200 352/352 [==============================] - 2s 5ms/step - loss: 0.7971 - val_loss: 0.7499 Epoch 16/200 352/352 [==============================] - 2s 5ms/step - loss: 0.7784 - val_loss: 0.7340 Epoch 17/200 352/352 [==============================] - 2s 5ms/step - loss: 0.7601 - val_loss: 0.7181 Epoch 18/200 352/352 [==============================] - 2s 5ms/step - loss: 0.7421 - val_loss: 0.7023 Epoch 19/200 352/352 [==============================] - 2s 5ms/step - loss: 0.7245 - val_loss: 0.6878 Epoch 20/200 352/352 [==============================] - 2s 5ms/step - loss: 0.7072 - val_loss: 0.6730 Epoch 21/200 352/352 [==============================] - 2s 5ms/step - loss: 0.6903 - val_loss: 0.6585 Epoch 22/200 352/352 [==============================] - 2s 5ms/step - loss: 0.6740 - val_loss: 0.6447 Epoch 23/200 352/352 [==============================] - 2s 5ms/step - loss: 0.6575 - val_loss: 0.6308 Epoch 24/200 352/352 [==============================] - 2s 5ms/step - loss: 0.6415 - val_loss: 0.6170 Epoch 25/200 352/352 [==============================] - 2s 5ms/step - loss: 0.6259 - val_loss: 0.6036 Epoch 26/200 352/352 [==============================] - 2s 5ms/step - loss: 0.6107 - val_loss: 0.5903 Epoch 27/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5964 - val_loss: 0.5784 Epoch 28/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5817 - val_loss: 0.5657 Epoch 29/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5674 - val_loss: 0.5548 Epoch 30/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5538 - val_loss: 0.5430 Epoch 31/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5400 - val_loss: 0.5298 Epoch 32/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5273 - val_loss: 0.5182 Epoch 33/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5155 - val_loss: 0.5073 Epoch 34/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5024 - val_loss: 0.4965 Epoch 35/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4887 - val_loss: 0.4861 Epoch 36/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4780 - val_loss: 0.4775 Epoch 37/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4659 - val_loss: 0.4638 Epoch 38/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4552 - val_loss: 0.4538 Epoch 39/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4413 - val_loss: 0.4431 Epoch 40/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4304 - val_loss: 0.4357 Epoch 41/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4218 - val_loss: 0.4322 Epoch 42/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4137 - val_loss: 0.4132 Epoch 43/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3981 - val_loss: 0.4045 Epoch 44/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3871 - val_loss: 0.3939 Epoch 45/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3775 - val_loss: 0.3873 Epoch 46/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3682 - val_loss: 0.3779 Epoch 47/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3596 - val_loss: 0.3683 Epoch 48/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3488 - val_loss: 0.3591 Epoch 49/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3397 - val_loss: 0.3505 Epoch 50/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3312 - val_loss: 0.3432 Epoch 51/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3240 - val_loss: 0.3348 Epoch 52/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3158 - val_loss: 0.3288 Epoch 53/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3063 - val_loss: 0.3224 Epoch 54/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2990 - val_loss: 0.3120 Epoch 55/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2906 - val_loss: 0.3064 Epoch 56/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2828 - val_loss: 0.2977 Epoch 57/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2750 - val_loss: 0.2908 Epoch 58/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2687 - val_loss: 0.2837 Epoch 59/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2609 - val_loss: 0.2770 Epoch 60/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2542 - val_loss: 0.2710 Epoch 61/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2479 - val_loss: 0.2645 Epoch 62/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2409 - val_loss: 0.2597 Epoch 63/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2346 - val_loss: 0.2521 Epoch 64/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2283 - val_loss: 0.2459 Epoch 65/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2227 - val_loss: 0.2417 Epoch 66/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2171 - val_loss: 0.2350 Epoch 67/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2103 - val_loss: 0.2290 Epoch 68/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2047 - val_loss: 0.2235 Epoch 69/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1992 - val_loss: 0.2182 Epoch 70/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1945 - val_loss: 0.2144 Epoch 71/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1892 - val_loss: 0.2080 Epoch 72/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1839 - val_loss: 0.2041 Epoch 73/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1794 - val_loss: 0.1986 Epoch 74/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1741 - val_loss: 0.1935 Epoch 75/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1696 - val_loss: 0.1889 Epoch 76/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1651 - val_loss: 0.1845 Epoch 77/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1609 - val_loss: 0.1801 Epoch 78/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1564 - val_loss: 0.1758 Epoch 79/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1521 - val_loss: 0.1715 Epoch 80/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1480 - val_loss: 0.1677 Epoch 81/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1443 - val_loss: 0.1636 Epoch 82/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1402 - val_loss: 0.1595 Epoch 83/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1367 - val_loss: 0.1557 Epoch 84/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1327 - val_loss: 0.1519 Epoch 85/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1292 - val_loss: 0.1483 Epoch 86/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1258 - val_loss: 0.1450 Epoch 87/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1225 - val_loss: 0.1421 Epoch 88/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1193 - val_loss: 0.1379 Epoch 89/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1158 - val_loss: 0.1345 Epoch 90/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1128 - val_loss: 0.1313 Epoch 91/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1098 - val_loss: 0.1284 Epoch 92/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1068 - val_loss: 0.1253 Epoch 93/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1040 - val_loss: 0.1222 Epoch 94/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1011 - val_loss: 0.1192 Epoch 95/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0985 - val_loss: 0.1166 Epoch 96/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0958 - val_loss: 0.1137 Epoch 97/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0931 - val_loss: 0.1110 Epoch 98/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0908 - val_loss: 0.1082 Epoch 99/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0883 - val_loss: 0.1056 Epoch 100/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0858 - val_loss: 0.1030 Epoch 101/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0835 - val_loss: 0.1006 Epoch 102/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0813 - val_loss: 0.0982 Epoch 103/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0792 - val_loss: 0.0960 Epoch 104/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0770 - val_loss: 0.0936 Epoch 105/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0749 - val_loss: 0.0913 Epoch 106/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0729 - val_loss: 0.0892 Epoch 107/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0709 - val_loss: 0.0871 Epoch 108/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0691 - val_loss: 0.0849 Epoch 109/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0672 - val_loss: 0.0829 Epoch 110/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0654 - val_loss: 0.0810 Epoch 111/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0636 - val_loss: 0.0790 Epoch 112/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0619 - val_loss: 0.0771 Epoch 113/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0602 - val_loss: 0.0753 Epoch 114/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0587 - val_loss: 0.0734 Epoch 115/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0571 - val_loss: 0.0716 Epoch 116/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0555 - val_loss: 0.0700 Epoch 117/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0540 - val_loss: 0.0683 Epoch 118/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0526 - val_loss: 0.0667 Epoch 119/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0512 - val_loss: 0.0650 Epoch 120/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0498 - val_loss: 0.0635 Epoch 121/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0485 - val_loss: 0.0621 Epoch 122/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0472 - val_loss: 0.0605 Epoch 123/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0459 - val_loss: 0.0591 Epoch 124/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0447 - val_loss: 0.0577 Epoch 125/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0435 - val_loss: 0.0563 Epoch 126/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0423 - val_loss: 0.0550 Epoch 127/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0412 - val_loss: 0.0536 Epoch 128/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0401 - val_loss: 0.0524 Epoch 129/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0390 - val_loss: 0.0511 Epoch 130/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0380 - val_loss: 0.0500 Epoch 131/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0370 - val_loss: 0.0487 Epoch 132/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0360 - val_loss: 0.0476 Epoch 133/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0350 - val_loss: 0.0465 Epoch 134/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0341 - val_loss: 0.0454 Epoch 135/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0332 - val_loss: 0.0443 Epoch 136/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0323 - val_loss: 0.0433 Epoch 137/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0314 - val_loss: 0.0422 Epoch 138/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0306 - val_loss: 0.0412 Epoch 139/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0298 - val_loss: 0.0403 Epoch 140/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0290 - val_loss: 0.0394 Epoch 141/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0282 - val_loss: 0.0384 Epoch 142/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0275 - val_loss: 0.0375 Epoch 143/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0267 - val_loss: 0.0366 Epoch 144/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0260 - val_loss: 0.0358 Epoch 145/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0253 - val_loss: 0.0349 Epoch 146/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0247 - val_loss: 0.0341 Epoch 147/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0240 - val_loss: 0.0333 Epoch 148/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0234 - val_loss: 0.0325 Epoch 149/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0227 - val_loss: 0.0318 Epoch 150/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0221 - val_loss: 0.0310 Epoch 151/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0216 - val_loss: 0.0303 Epoch 152/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0210 - val_loss: 0.0296 Epoch 153/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0204 - val_loss: 0.0289 Epoch 154/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0199 - val_loss: 0.0283 Epoch 155/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0194 - val_loss: 0.0276 Epoch 156/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0189 - val_loss: 0.0269 Epoch 157/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0184 - val_loss: 0.0263 Epoch 158/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0179 - val_loss: 0.0257 Epoch 159/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0174 - val_loss: 0.0251 Epoch 160/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0169 - val_loss: 0.0245 Epoch 161/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0165 - val_loss: 0.0240 Epoch 162/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0161 - val_loss: 0.0234 Epoch 163/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0157 - val_loss: 0.0229 Epoch 164/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0152 - val_loss: 0.0223 Epoch 165/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0148 - val_loss: 0.0218 Epoch 166/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0144 - val_loss: 0.0213 Epoch 167/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0141 - val_loss: 0.0208 Epoch 168/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0137 - val_loss: 0.0204 Epoch 169/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0133 - val_loss: 0.0199 Epoch 170/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0130 - val_loss: 0.0194 Epoch 171/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0127 - val_loss: 0.0190 Epoch 172/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0123 - val_loss: 0.0186 Epoch 173/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0120 - val_loss: 0.0181 Epoch 174/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0117 - val_loss: 0.0177 Epoch 175/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0114 - val_loss: 0.0173 Epoch 176/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0111 - val_loss: 0.0169 Epoch 177/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0108 - val_loss: 0.0165 Epoch 178/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0105 - val_loss: 0.0162 Epoch 179/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0103 - val_loss: 0.0158 Epoch 180/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0100 - val_loss: 0.0154 Epoch 181/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0097 - val_loss: 0.0151 Epoch 182/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0095 - val_loss: 0.0147 Epoch 183/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0092 - val_loss: 0.0144 Epoch 184/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0090 - val_loss: 0.0141 Epoch 185/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0088 - val_loss: 0.0138 Epoch 186/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0085 - val_loss: 0.0135 Epoch 187/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0083 - val_loss: 0.0132 Epoch 188/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0081 - val_loss: 0.0129 Epoch 189/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0079 - val_loss: 0.0126 Epoch 190/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0077 - val_loss: 0.0123 Epoch 191/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0075 - val_loss: 0.0120 Epoch 192/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0073 - val_loss: 0.0117 Epoch 193/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0071 - val_loss: 0.0115 Epoch 194/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0069 - val_loss: 0.0112 Epoch 195/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0068 - val_loss: 0.0110 Epoch 196/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0066 - val_loss: 0.0107 Epoch 197/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0064 - val_loss: 0.0105 Epoch 198/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0062 - val_loss: 0.0103 Epoch 199/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0061 - val_loss: 0.0100 Epoch 200/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0059 - val_loss: 0.0098
In [ ]:
Copied!
Pc2, _ = pmodel.color_matrix(X2, L2, p_th=p_threshold2)
y_colors = pmodel.verify_colors(Pc2)
Pc2, _ = pmodel.color_matrix(X2, L2, p_th=p_threshold2)
y_colors = pmodel.verify_colors(Pc2)
In [ ]:
Copied!
analyzer = cm.analyze_reconstruction(cf_model,
X=(R2, T2),
L=L2,
Pc=Pc2,
p_threshold=p_threshold2, policy_threshold=policy_threshold)
highlight("Reestimate the entire rating matrix (X) with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=False)
highlight("Reestimate ONLY the unreliable entries in X with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=True, verbose=2)
analyzer = cm.analyze_reconstruction(cf_model,
X=(R2, T2),
L=L2,
Pc=Pc2,
p_threshold=p_threshold2, policy_threshold=policy_threshold)
highlight("Reestimate the entire rating matrix (X) with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=False)
highlight("Reestimate ONLY the unreliable entries in X with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=True, verbose=2)
================================================================================ Reestimate the entire rating matrix (X) with learned latent factors/embeddings ================================================================================ [info] From R to Rh, delta(Frobenius norm)= 41.973287210828786 [info] From T to Th, delta(Frobenius norm)= 17.698433855773427 [info] How different are lh and lh_new? 0.0 [result] Majority vote: F1 score with the original T: 0.18543046357615894 [result] Majority vote: F1 score with re-estimated Th using original p_threshold: 0.18543046357615894 [result] Majority vote: F1 score with re-estimated Th: 0.18543046357615894 [result] Stacking: F1 score with the original T: 0.18543046357615894 [result] Stacking: F1 score with re-estimated Th: 0.18543046357615894 [result] Best settings (complete): lh_maxvote, score: 0.18543046357615894 ================================================================================ Reestimate ONLY the unreliable entries in X with learned latent factors/embeddings ================================================================================ [info] From R to Rh, delta(Frobenius norm)= 32.96501379339972 [info] From T to Th, delta(Frobenius norm)= 0.10108785286424554 [info] How different are lh and lh_new? 0.0 [result] Majority vote: F1 score with the original T: 0.18543046357615894 [result] Majority vote: F1 score with re-estimated Th using original p_threshold: 0.18543046357615894 [result] Majority vote: F1 score with re-estimated Th: 0.18543046357615894 [result] Stacking: F1 score with the original T: 0.18543046357615894 [result] Stacking: F1 score with re-estimated Th: 0.18543046357615894 [result] Methods ranked: [(0.18543046357615894, 'lh_maxvote'), (0.18543046357615894, 'lh2_maxvote_pth_unadjusted'), (0.18543046357615894, 'lh2_maxvote_pth_adjusted')] [result] Best settings (unreliable only): lh_maxvote, score: 0.18543046357615894 [help] Reestiamted quantities are available through the following keys: - ratings - p_threshold2 - lh_maxvote - lh2_maxvote_pth_unadjusted - lh2_maxvote_pth_adjusted - score_lh_maxvote - score_baseline - score_lh2_maxvote_pth_unadjusted - score_lh2_maxvote_pth_adjusted - score_lh_stacker - score_lh2_stacker_pth_adjusted - best_params - best_params_score
Observation: The model at second CF-stacking iteration seems to have been "saturated" without being able to improve further
Oracle: What if somehow we knew the true labels?¶
- What's the max performance we can possibly get given a particular loss function?
In [ ]:
Copied!
lh3 = L_test
L3 = np.hstack((L_train, L_test))
X3 = np.hstack((R, T)) # X3 is really identical to X (as we are using the original rating matrices)
lh3 = L_test
L3 = np.hstack((L_train, L_test))
X3 = np.hstack((R, T)) # X3 is really identical to X (as we are using the original rating matrices)
In [ ]:
Copied!
# workflow: p_threshold -> lh -> color matrix
Pc_true, _ = pmodel.color_matrix(T, L_test, p_threshold) # Mc: Color matrix evaluated via estimated labels
Pf_true = pmodel.to_preference(Pc_true, neutral=0.0)
metrics = pmodel.eval_estimated_probability_filter(Pf_true, T, L_test, p_threshold, eps=1e-3)
highlight("Guesstimated labeling (on T) via TRUE LABELS")
print(f"> Labeling accuracy: {1.0}")
print(f"> Reliable-to-correct ratio: {metrics['p_overlap']}") # Fraction of entries predicted reliable and are actually correct (TPs or TNs)
print(f"> Precision: {metrics['precision']}, Recall: {metrics['recall']}")
print(f"> Precision(TP): {metrics['precision_tp']}, Recall(TP): {metrics['recall_tp']} => f1(TP): {f_score(metrics['precision_tp'], metrics['recall_tp'])}")
# NOTE: Precision(TP) is defined as P(TP|reliable), i.e. probability of being TP given predicted "reliable" (estimate in an average sense)
print(f"> Error rate: {metrics['p_missed']}") # Probability of predicting reliable but hitting either FPs or FNs
# workflow: p_threshold -> lh -> color matrix
Pc_true, _ = pmodel.color_matrix(T, L_test, p_threshold) # Mc: Color matrix evaluated via estimated labels
Pf_true = pmodel.to_preference(Pc_true, neutral=0.0)
metrics = pmodel.eval_estimated_probability_filter(Pf_true, T, L_test, p_threshold, eps=1e-3)
highlight("Guesstimated labeling (on T) via TRUE LABELS")
print(f"> Labeling accuracy: {1.0}")
print(f"> Reliable-to-correct ratio: {metrics['p_overlap']}") # Fraction of entries predicted reliable and are actually correct (TPs or TNs)
print(f"> Precision: {metrics['precision']}, Recall: {metrics['recall']}")
print(f"> Precision(TP): {metrics['precision_tp']}, Recall(TP): {metrics['recall_tp']} => f1(TP): {f_score(metrics['precision_tp'], metrics['recall_tp'])}")
# NOTE: Precision(TP) is defined as P(TP|reliable), i.e. probability of being TP given predicted "reliable" (estimate in an average sense)
print(f"> Error rate: {metrics['p_missed']}") # Probability of predicting reliable but hitting either FPs or FNs
================================================================================ Guesstimated labeling (on T) via TRUE LABELS ================================================================================ > Labeling accuracy: 1.0 > Reliable-to-correct ratio: 1.0 > Precision: 0.9999996398993014, Recall: 0.9999996398993014 > Precision(TP): 0.14584078291653477, Recall(TP): 0.9999975308702942 => f1(TP): 0.25455672246591926 > Error rate: 0.0
In [ ]:
Copied!
n_users, n_items = X.shape
# Uncomment previously-defined parameters below if needed
# fold_number = 0
# test_size = 0.1
# policy_threshold = 'fmax'
# conf_measure = 'brier'
# n_factors = 100
# alpha = 100
# lr = 0.001
# batch_size = 64
# target_type = 'label' # Options: 'generic', 'rating', 'label'
# conf_type = 'Cn' # Options: 'Cn', 'Cw', 'C0', or your customized weights
epochs = 200
# Note: confidence_weighted_loss converges relatively faster compared to MSE, BCE etc.
loss_fn = tf.keras.losses.BinaryCrossentropy() # Options: cm.confidence_weighted_loss, cm.c_squared_loss, tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.MeanSquaredError(), ...
cf_model = cm.get_cfnet_compiled(n_users, n_items, n_factors, loss=loss_fn, lr=lr)
cf_model = cm.training_pipeline(input_model=(cf_model, loss_fn),
input_data=(R, T, U, L_train, ),
lh = L_test, # use true labels here and see what happens
# Should we combine R and T into a single matrix X? Set to True if so
is_cascade = True, # Set to True here to combine R and T into X
# SGD optimization parameters
test_size = test_size,
epochs = epochs,
batch_size=batch_size,
# CF hyperparameters
# n_factors=n_factors, # this is factored into model definition
alpha=alpha,
conf_measure=conf_measure,
policy_threshold=policy_threshold,
conf_type=conf_type, # default sparse confidence matrix (Cn)
# target_type=target_type,
fold_number=fold_number)
n_users, n_items = X.shape
# Uncomment previously-defined parameters below if needed
# fold_number = 0
# test_size = 0.1
# policy_threshold = 'fmax'
# conf_measure = 'brier'
# n_factors = 100
# alpha = 100
# lr = 0.001
# batch_size = 64
# target_type = 'label' # Options: 'generic', 'rating', 'label'
# conf_type = 'Cn' # Options: 'Cn', 'Cw', 'C0', or your customized weights
epochs = 200
# Note: confidence_weighted_loss converges relatively faster compared to MSE, BCE etc.
loss_fn = tf.keras.losses.BinaryCrossentropy() # Options: cm.confidence_weighted_loss, cm.c_squared_loss, tf.keras.losses.BinaryCrossentropy(), tf.keras.losses.MeanSquaredError(), ...
cf_model = cm.get_cfnet_compiled(n_users, n_items, n_factors, loss=loss_fn, lr=lr)
cf_model = cm.training_pipeline(input_model=(cf_model, loss_fn),
input_data=(R, T, U, L_train, ),
lh = L_test, # use true labels here and see what happens
# Should we combine R and T into a single matrix X? Set to True if so
is_cascade = True, # Set to True here to combine R and T into X
# SGD optimization parameters
test_size = test_size,
epochs = epochs,
batch_size=batch_size,
# CF hyperparameters
# n_factors=n_factors, # this is factored into model definition
alpha=alpha,
conf_measure=conf_measure,
policy_threshold=policy_threshold,
conf_type=conf_type, # default sparse confidence matrix (Cn)
# target_type=target_type,
fold_number=fold_number)
[merge] Merging 'L_train' and 'lh': len(L_train): 3750 || len(lh): 1250 => len(L): 5000 [merge] Merging 'R' and 'T': shape(R):(5, 3750) || shape(T): (5, 1250) => shape(X): (5, 5000) (make_cn) Using WEIGHTED confidence matrix to approximate ratings ... [info] Confidence matrix type: Cn, target data type: label Epoch 1/200 352/352 [==============================] - 3s 7ms/step - loss: 2.9402 - val_loss: 3.0462 Epoch 2/200 352/352 [==============================] - 2s 5ms/step - loss: 2.3761 - val_loss: 2.2893 Epoch 3/200 352/352 [==============================] - 2s 5ms/step - loss: 1.8277 - val_loss: 1.8937 Epoch 4/200 352/352 [==============================] - 2s 5ms/step - loss: 1.1460 - val_loss: 1.3542 Epoch 5/200 352/352 [==============================] - 2s 5ms/step - loss: 0.8436 - val_loss: 1.2068 Epoch 6/200 352/352 [==============================] - 2s 5ms/step - loss: 0.7082 - val_loss: 1.1293 Epoch 7/200 352/352 [==============================] - 2s 5ms/step - loss: 0.6478 - val_loss: 1.0859 Epoch 8/200 352/352 [==============================] - 2s 5ms/step - loss: 0.6216 - val_loss: 1.0645 Epoch 9/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5993 - val_loss: 1.0596 Epoch 10/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5870 - val_loss: 1.0295 Epoch 11/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5718 - val_loss: 1.0077 Epoch 12/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5535 - val_loss: 1.0071 Epoch 13/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5389 - val_loss: 0.9809 Epoch 14/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5301 - val_loss: 0.9474 Epoch 15/200 352/352 [==============================] - 2s 5ms/step - loss: 0.5109 - val_loss: 0.9313 Epoch 16/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4927 - val_loss: 0.9156 Epoch 17/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4808 - val_loss: 0.8983 Epoch 18/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4629 - val_loss: 0.8782 Epoch 19/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4480 - val_loss: 0.8547 Epoch 20/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4335 - val_loss: 0.8358 Epoch 21/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4193 - val_loss: 0.8221 Epoch 22/200 352/352 [==============================] - 2s 5ms/step - loss: 0.4082 - val_loss: 0.8093 Epoch 23/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3990 - val_loss: 0.7979 Epoch 24/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3908 - val_loss: 0.7873 Epoch 25/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3833 - val_loss: 0.7761 Epoch 26/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3760 - val_loss: 0.7652 Epoch 27/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3691 - val_loss: 0.7542 Epoch 28/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3623 - val_loss: 0.7433 Epoch 29/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3558 - val_loss: 0.7329 Epoch 30/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3493 - val_loss: 0.7213 Epoch 31/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3427 - val_loss: 0.7104 Epoch 32/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3365 - val_loss: 0.6993 Epoch 33/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3306 - val_loss: 0.6885 Epoch 34/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3245 - val_loss: 0.6778 Epoch 35/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3186 - val_loss: 0.6676 Epoch 36/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3136 - val_loss: 0.6581 Epoch 37/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3082 - val_loss: 0.6476 Epoch 38/200 352/352 [==============================] - 2s 5ms/step - loss: 0.3018 - val_loss: 0.6364 Epoch 39/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2968 - val_loss: 0.6269 Epoch 40/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2910 - val_loss: 0.6182 Epoch 41/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2870 - val_loss: 0.6069 Epoch 42/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2803 - val_loss: 0.5974 Epoch 43/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2752 - val_loss: 0.5878 Epoch 44/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2701 - val_loss: 0.5793 Epoch 45/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2656 - val_loss: 0.5717 Epoch 46/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2606 - val_loss: 0.5607 Epoch 47/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2554 - val_loss: 0.5518 Epoch 48/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2505 - val_loss: 0.5444 Epoch 49/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2469 - val_loss: 0.5335 Epoch 50/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2407 - val_loss: 0.5248 Epoch 51/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2379 - val_loss: 0.5167 Epoch 52/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2319 - val_loss: 0.5085 Epoch 53/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2281 - val_loss: 0.5000 Epoch 54/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2233 - val_loss: 0.4952 Epoch 55/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2201 - val_loss: 0.4846 Epoch 56/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2148 - val_loss: 0.4765 Epoch 57/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2106 - val_loss: 0.4682 Epoch 58/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2061 - val_loss: 0.4605 Epoch 59/200 352/352 [==============================] - 2s 5ms/step - loss: 0.2027 - val_loss: 0.4535 Epoch 60/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1989 - val_loss: 0.4462 Epoch 61/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1949 - val_loss: 0.4383 Epoch 62/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1912 - val_loss: 0.4314 Epoch 63/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1875 - val_loss: 0.4254 Epoch 64/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1846 - val_loss: 0.4175 Epoch 65/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1800 - val_loss: 0.4110 Epoch 66/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1767 - val_loss: 0.4082 Epoch 67/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1739 - val_loss: 0.3973 Epoch 68/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1696 - val_loss: 0.3908 Epoch 69/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1664 - val_loss: 0.3844 Epoch 70/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1638 - val_loss: 0.3782 Epoch 71/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1600 - val_loss: 0.3724 Epoch 72/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1567 - val_loss: 0.3657 Epoch 73/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1536 - val_loss: 0.3598 Epoch 74/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1507 - val_loss: 0.3546 Epoch 75/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1480 - val_loss: 0.3490 Epoch 76/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1447 - val_loss: 0.3423 Epoch 77/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1420 - val_loss: 0.3368 Epoch 78/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1396 - val_loss: 0.3312 Epoch 79/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1362 - val_loss: 0.3260 Epoch 80/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1335 - val_loss: 0.3204 Epoch 81/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1309 - val_loss: 0.3151 Epoch 82/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1282 - val_loss: 0.3107 Epoch 83/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1261 - val_loss: 0.3048 Epoch 84/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1231 - val_loss: 0.3000 Epoch 85/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1206 - val_loss: 0.2951 Epoch 86/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1183 - val_loss: 0.2903 Epoch 87/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1160 - val_loss: 0.2852 Epoch 88/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1138 - val_loss: 0.2807 Epoch 89/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1113 - val_loss: 0.2761 Epoch 90/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1094 - val_loss: 0.2721 Epoch 91/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1069 - val_loss: 0.2677 Epoch 92/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1047 - val_loss: 0.2627 Epoch 93/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1025 - val_loss: 0.2584 Epoch 94/200 352/352 [==============================] - 2s 5ms/step - loss: 0.1004 - val_loss: 0.2541 Epoch 95/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0988 - val_loss: 0.2506 Epoch 96/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0966 - val_loss: 0.2458 Epoch 97/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0944 - val_loss: 0.2417 Epoch 98/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0925 - val_loss: 0.2379 Epoch 99/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0907 - val_loss: 0.2341 Epoch 100/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0889 - val_loss: 0.2302 Epoch 101/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0872 - val_loss: 0.2271 Epoch 102/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0854 - val_loss: 0.2230 Epoch 103/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0836 - val_loss: 0.2191 Epoch 104/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0818 - val_loss: 0.2158 Epoch 105/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0802 - val_loss: 0.2120 Epoch 106/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0785 - val_loss: 0.2086 Epoch 107/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0770 - val_loss: 0.2055 Epoch 108/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0754 - val_loss: 0.2020 Epoch 109/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0738 - val_loss: 0.1987 Epoch 110/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0723 - val_loss: 0.1956 Epoch 111/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0708 - val_loss: 0.1923 Epoch 112/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0694 - val_loss: 0.1891 Epoch 113/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0680 - val_loss: 0.1865 Epoch 114/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0666 - val_loss: 0.1831 Epoch 115/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0652 - val_loss: 0.1803 Epoch 116/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0639 - val_loss: 0.1773 Epoch 117/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0625 - val_loss: 0.1745 Epoch 118/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0613 - val_loss: 0.1717 Epoch 119/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0600 - val_loss: 0.1691 Epoch 120/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0588 - val_loss: 0.1663 Epoch 121/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0576 - val_loss: 0.1636 Epoch 122/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0564 - val_loss: 0.1610 Epoch 123/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0552 - val_loss: 0.1585 Epoch 124/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0541 - val_loss: 0.1561 Epoch 125/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0530 - val_loss: 0.1535 Epoch 126/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0519 - val_loss: 0.1510 Epoch 127/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0508 - val_loss: 0.1486 Epoch 128/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0498 - val_loss: 0.1464 Epoch 129/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0488 - val_loss: 0.1440 Epoch 130/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0477 - val_loss: 0.1417 Epoch 131/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0467 - val_loss: 0.1396 Epoch 132/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0458 - val_loss: 0.1373 Epoch 133/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0449 - val_loss: 0.1352 Epoch 134/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0439 - val_loss: 0.1331 Epoch 135/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0430 - val_loss: 0.1310 Epoch 136/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0422 - val_loss: 0.1290 Epoch 137/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0413 - val_loss: 0.1270 Epoch 138/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0404 - val_loss: 0.1251 Epoch 139/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0396 - val_loss: 0.1231 Epoch 140/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0388 - val_loss: 0.1212 Epoch 141/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0380 - val_loss: 0.1192 Epoch 142/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0372 - val_loss: 0.1175 Epoch 143/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0365 - val_loss: 0.1158 Epoch 144/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0357 - val_loss: 0.1139 Epoch 145/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0349 - val_loss: 0.1123 Epoch 146/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0342 - val_loss: 0.1106 Epoch 147/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0335 - val_loss: 0.1089 Epoch 148/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0328 - val_loss: 0.1072 Epoch 149/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0321 - val_loss: 0.1056 Epoch 150/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0315 - val_loss: 0.1041 Epoch 151/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0308 - val_loss: 0.1025 Epoch 152/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0302 - val_loss: 0.1009 Epoch 153/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0296 - val_loss: 0.0994 Epoch 154/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0290 - val_loss: 0.0980 Epoch 155/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0284 - val_loss: 0.0965 Epoch 156/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0278 - val_loss: 0.0951 Epoch 157/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0272 - val_loss: 0.0937 Epoch 158/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0266 - val_loss: 0.0924 Epoch 159/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0261 - val_loss: 0.0910 Epoch 160/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0256 - val_loss: 0.0897 Epoch 161/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0250 - val_loss: 0.0885 Epoch 162/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0245 - val_loss: 0.0871 Epoch 163/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0240 - val_loss: 0.0858 Epoch 164/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0235 - val_loss: 0.0847 Epoch 165/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0230 - val_loss: 0.0834 Epoch 166/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0226 - val_loss: 0.0822 Epoch 167/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0221 - val_loss: 0.0810 Epoch 168/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0216 - val_loss: 0.0798 Epoch 169/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0212 - val_loss: 0.0787 Epoch 170/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0207 - val_loss: 0.0776 Epoch 171/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0203 - val_loss: 0.0766 Epoch 172/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0199 - val_loss: 0.0755 Epoch 173/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0195 - val_loss: 0.0744 Epoch 174/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0191 - val_loss: 0.0734 Epoch 175/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0187 - val_loss: 0.0724 Epoch 176/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0183 - val_loss: 0.0714 Epoch 177/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0179 - val_loss: 0.0704 Epoch 178/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0176 - val_loss: 0.0694 Epoch 179/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0172 - val_loss: 0.0686 Epoch 180/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0169 - val_loss: 0.0676 Epoch 181/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0165 - val_loss: 0.0667 Epoch 182/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0162 - val_loss: 0.0658 Epoch 183/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0158 - val_loss: 0.0649 Epoch 184/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0155 - val_loss: 0.0641 Epoch 185/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0152 - val_loss: 0.0632 Epoch 186/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0149 - val_loss: 0.0624 Epoch 187/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0146 - val_loss: 0.0615 Epoch 188/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0143 - val_loss: 0.0607 Epoch 189/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0140 - val_loss: 0.0599 Epoch 190/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0137 - val_loss: 0.0592 Epoch 191/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0134 - val_loss: 0.0583 Epoch 192/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0131 - val_loss: 0.0576 Epoch 193/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0129 - val_loss: 0.0569 Epoch 194/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0126 - val_loss: 0.0562 Epoch 195/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0124 - val_loss: 0.0554 Epoch 196/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0121 - val_loss: 0.0547 Epoch 197/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0119 - val_loss: 0.0541 Epoch 198/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0116 - val_loss: 0.0534 Epoch 199/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0114 - val_loss: 0.0528 Epoch 200/200 352/352 [==============================] - 2s 5ms/step - loss: 0.0111 - val_loss: 0.0521
In [ ]:
Copied!
Pc3, _ = pmodel.color_matrix(X3, L3, p_th=p_threshold) # L3 combines L_train and L_test (true labels); X3 is same as X = [R|T]
analyzer = cm.analyze_reconstruction(cf_model,
X=(R, T),
L=(L_train, L_test),
Pc=Pc3,
p_threshold=p_threshold, policy_threshold=policy_threshold)
highlight("Reestimate the entire rating matrix (X) with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=False)
highlight("Reestimate ONLY the unreliable entries in X with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=True, verbose=2)
Pc3, _ = pmodel.color_matrix(X3, L3, p_th=p_threshold) # L3 combines L_train and L_test (true labels); X3 is same as X = [R|T]
analyzer = cm.analyze_reconstruction(cf_model,
X=(R, T),
L=(L_train, L_test),
Pc=Pc3,
p_threshold=p_threshold, policy_threshold=policy_threshold)
highlight("Reestimate the entire rating matrix (X) with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=False)
highlight("Reestimate ONLY the unreliable entries in X with learned latent factors/embeddings")
reestimated = analyzer(L_test, unreliable_only=True, verbose=2)
================================================================================ Reestimate the entire rating matrix (X) with learned latent factors/embeddings ================================================================================ [info] From R to Rh, delta(Frobenius norm)= 74.8732567192456 [info] From T to Th, delta(Frobenius norm)= 41.840877098532076 [info] How different are lh and lh_new? 0.6424 [result] Majority vote: F1 score with the original T: 0.18981018981018982 [result] Majority vote: F1 score with re-estimated Th using original p_threshold: 0.19364161849710984 [result] Majority vote: F1 score with re-estimated Th: 0.7927927927927928 [result] Stacking: F1 score with the original T: 0.11188811188811189 [result] Stacking: F1 score with re-estimated Th: 1.0 [result] Best settings (complete): lh2_maxvote_pth_adjusted, score: 0.7927927927927928 ================================================================================ Reestimate ONLY the unreliable entries in X with learned latent factors/embeddings ================================================================================ [info] From R to Rh, delta(Frobenius norm)= 68.2787577146574 [info] From T to Th, delta(Frobenius norm)= 35.23255020865507 [info] How different are lh and lh_new? 0.656 [result] Majority vote: F1 score with the original T: 0.18981018981018982 [result] Majority vote: F1 score with re-estimated Th using original p_threshold: 0.44592346089850254 [result] Majority vote: F1 score with re-estimated Th: 0.9652509652509652 [result] Stacking: F1 score with the original T: 0.11188811188811189 [result] Stacking: F1 score with re-estimated Th: 1.0 [result] Methods ranked: [(0.9652509652509652, 'lh2_maxvote_pth_adjusted'), (0.44592346089850254, 'lh2_maxvote_pth_unadjusted'), (0.18981018981018982, 'lh_maxvote')] [result] Best settings (unreliable only): lh2_maxvote_pth_adjusted, score: 0.9652509652509652 [help] Reestiamted quantities are available through the following keys: - ratings - p_threshold2 - lh_maxvote - lh2_maxvote_pth_unadjusted - lh2_maxvote_pth_adjusted - score_lh_maxvote - score_baseline - score_lh2_maxvote_pth_unadjusted - score_lh2_maxvote_pth_adjusted - score_lh_stacker - score_lh2_stacker_pth_adjusted - best_params - best_params_score
In [ ]:
Copied!