The AL algorithms didn't work

Question

The AL algorithms didn't work

evanzhu2013 opened this issue 6 years ago · comments

Thanks so much for your great work.

It seems that the AL algorithms had no better score. Compared to random sampling , the Uncertainty and QBD had no better accuracy. The datasets are sklearn's load_breast_cancer.

Besides, the random_query has one more point with the same stoping criterion.

Looking forward your reply.

Code is shown below :

# coding: utf-8

from sklearn.datasets import load_breast_cancer
# Ploting
from matplotlib.pylab import plt
get_ipython().run_line_magic('matplotlib', 'inline')
get_ipython().run_line_magic('config', "InlineBackend.figure_format = 'retina'")
from sklearn.ensemble import RandomForestClassifier

import xgboost
import pandas as pd
import warnings
warnings.filterwarnings('ignore')
from sklearn.datasets import load_iris,load_breast_cancer
from alipy.experiment.al_experiment import AlExperiment

import copy
from sklearn.datasets import make_classification
from alipy import ToolBox
from alipy.query_strategy.query_labels import QueryInstanceGraphDensity, QueryInstanceQBC,     QueryInstanceQUIRE, QueryRandom, QueryInstanceUncertainty, QureyExpectedErrorReduction, QueryInstanceLAL

dataset = load_breast_cancer()
X = dataset.data
y = dataset.target
alibox = ToolBox(X=X, y=y, query_type='AllLabels', saving_path='.')

# Split data
alibox.split_AL(test_ratio=0.2, initial_label_rate=0.2, split_count=5)


model = RandomForestClassifier(n_estimators=100, max_depth=5,random_state=0)


stopping_criterion = alibox.get_stopping_criterion('num_of_queries', 8)
def main_loop(alibox, strategy, round):
    # Get the data split of one fold experiment
    train_idx, test_idx, label_ind, unlab_ind = alibox.get_split(round)
    # Get intermediate results saver for one fold experiment
    saver = alibox.get_stateio(round)
    while not stopping_criterion.is_stop():
        # Select a subset of Uind according to the query strategy
        # Passing model=None to use the default model for evaluating the committees' disagreement
        select_ind = strategy.select(label_ind, unlab_ind,batch_size=50)
#         print(len(select_ind))
        label_ind.update(select_ind)
        unlab_ind.difference_update(select_ind)

        # Update model and calc performance according to the model you are using
        model.fit(X=X[label_ind.index, :], y=y[label_ind.index])
        pred = model.predict(X[test_idx, :])
        accuracy = alibox.calc_performance_metric(y_true=y[test_idx],
                                                  y_pred=pred,
                                                  performance_metric='roc_auc_score')

        # Save intermediate results to file
        st = alibox.State(select_index=select_ind, performance=accuracy)
        saver.add_state(st)

        # Passing the current progress to stopping criterion object
        stopping_criterion.update_information(saver)
    # Reset the progress in stopping criterion object
    stopping_criterion.reset()
    return saver

unc_result = []
qbc_result = []
eer_result = []
quire_result = []
density_result = []


for round in range(3):
    train_idx, test_idx, label_ind, unlab_ind = alibox.get_split(round)

    # Use pre-defined strategy
    unc = QueryInstanceUncertainty(X, y)
    qbc = QueryInstanceQBC(X, y)

    unc_result.append(copy.deepcopy(main_loop(alibox, unc, round)))
    qbc_result.append(copy.deepcopy(main_loop(alibox, qbc, round)))


random = QueryRandom(X, y)
random_result = []

for round in range(3):
    # Get the data split of one fold experiment
    train_idx, test_idx, label_ind, unlab_ind = alibox.get_split(round)
    # Get intermediate results saver for one fold experiment
    saver = alibox.get_stateio(round)
    # calc the initial point
    model.fit(X=X[label_ind.index, :], y=y[label_ind.index])
    pred = model.predict(X[test_idx, :])
    accuracy = sum(pred == y[test_idx]) / len(test_idx)
    saver.set_initial_point(accuracy)

    while not stopping_criterion.is_stop():
        # Select a subset of Uind according to the query strategy
        # Passing model=None to use the default model for evaluating the committees' disagreement
        select_ind = random.select(unlab_ind, batch_size=50)
        label_ind.update(select_ind)
        unlab_ind.difference_update(select_ind)

        # Update model and calc performance according to the model you are using
        model.fit(X=X[label_ind.index, :], y=y[label_ind.index])
        pred = model.predict(X[test_idx, :])
        accuracy = alibox.calc_performance_metric(y_true=y[test_idx],
                                                y_pred=pred,
                                                performance_metric='roc_auc_score')

        # Save intermediate results to file
        st = alibox.State(select_index=select_ind, performance=accuracy)
        saver.add_state(st)
        saver.save()

        # Passing the current progress to stopping criterion object
        stopping_criterion.update_information(saver)
    # Reset the progress in stopping criterion object
    stopping_criterion.reset()
    random_result.append(copy.deepcopy(saver))
    
analyser = alibox.get_experiment_analyser(x_axis='num_of_queries')
analyser.add_method(method_name='Unc', method_results=unc_result)
analyser.add_method(method_name='Rnd', method_results=random_result)
analyser.add_method(method_name='QBC', method_results=qbc_result)

print(analyser)
analyser.plot_learning_curves(title='Example of alipy', std_area=False)

Tang · Answer 1 · Fri Mar 01 2019 15:59:16 GMT+0800 (China Standard Time)

Hi, evanzhu2013.

The reason that can cause the first issue can be plentiful. Such as:

The initially labeled ratio is too high.
The number of experiments with different data split is not enough.
The classifier can not fit the data well even with all the data in the current performance metric. (Try to use another model or dataset, or another kernel function if your model can be kernelized, or use another performance metric.)
The interval between 2 performance point is too small. (Try to add a state in every 3 queries or more.)

In your case, I think it is the performance metric problem.
You want to use accuracy but pass a performance_metric='roc_auc_score' when calculating the performance.

For the 2nd issue, the random_query has one more point with the same stoping criterion.
That is because you set the initial point for random strategy by saver.set_initial_point(accuracy) but not for uncertainty and QBC strategies.

Actually, you can use the same main_loop function for random strategy even in an old version of ALiPy, just replace the code select_ind = strategy.select(label_ind, unlab_ind,batch_size=50) with select_ind = strategy.select(label_index=label_ind, unlabel_index=unlab_ind, batch_size=50).

Here is an example modification of your code to tackle the issues:

# coding: utf-8

# Ploting
import warnings

from sklearn.ensemble import RandomForestClassifier

warnings.filterwarnings('ignore')
from sklearn.datasets import load_breast_cancer

import copy
from alipy import ToolBox
from alipy.query_strategy.query_labels import QueryInstanceQBC, QueryRandom, QueryInstanceUncertainty

dataset = load_breast_cancer()
X = dataset.data
y = dataset.target
alibox = ToolBox(X=X, y=y, query_type='AllLabels', saving_path='.')

# Split data
alibox.split_AL(test_ratio=0.2, initial_label_rate=0.2, split_count=5)

model = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)

stopping_criterion = alibox.get_stopping_criterion('num_of_queries', 8)


def main_loop(alibox, strategy, round):
    # Get the data split of one fold experiment
    train_idx, test_idx, label_ind, unlab_ind = alibox.get_split(round)
    # Get intermediate results saver for one fold experiment
    saver = alibox.get_stateio(round)
    model.fit(X=X[label_ind.index, :], y=y[label_ind.index])
    pred = model.predict(X[test_idx, :])
    accuracy = alibox.calc_performance_metric(y_true=y[test_idx],
                                              y_pred=pred,
                                              performance_metric='accuracy_score')
    saver.set_initial_point(accuracy)
    while not stopping_criterion.is_stop():
        # Select a subset of Uind according to the query strategy
        # Passing model=None to use the default model for evaluating the committees' disagreement
        select_ind = strategy.select(label_index=label_ind, unlabel_index=unlab_ind, batch_size=50)
        #         print(len(select_ind))
        label_ind.update(select_ind)
        unlab_ind.difference_update(select_ind)

        # Update model and calc performance according to the model you are using
        model.fit(X=X[label_ind.index, :], y=y[label_ind.index])
        pred = model.predict(X[test_idx, :])
        accuracy = alibox.calc_performance_metric(y_true=y[test_idx],
                                                  y_pred=pred,
                                                  performance_metric='accuracy_score')

        # Save intermediate results to file
        st = alibox.State(select_index=select_ind, performance=accuracy)
        saver.add_state(st)

        # Passing the current progress to stopping criterion object
        stopping_criterion.update_information(saver)
    # Reset the progress in stopping criterion object
    stopping_criterion.reset()
    return saver


unc_result = []
qbc_result = []
eer_result = []
quire_result = []
density_result = []

random_result = []

for round in range(3):
    train_idx, test_idx, label_ind, unlab_ind = alibox.get_split(round)

    # Use pre-defined strategy
    unc = QueryInstanceUncertainty(X, y)
    qbc = QueryInstanceQBC(X, y)
    random = QueryRandom(X, y)

    unc_result.append(copy.deepcopy(main_loop(alibox, unc, round)))
    qbc_result.append(copy.deepcopy(main_loop(alibox, qbc, round)))
    random_result.append(copy.deepcopy(main_loop(alibox, random, round)))

analyser = alibox.get_experiment_analyser(x_axis='num_of_queries')
analyser.add_method(method_name='Unc', method_results=unc_result)
analyser.add_method(method_name='Rnd', method_results=random_result)
analyser.add_method(method_name='QBC', method_results=qbc_result)

print(analyser)
analyser.plot_learning_curves(title='Example of alipy', std_area=False)

Evan Zhu · Answer 2 · Mon Mar 04 2019 14:43:59 GMT+0800 (China Standard Time)

Thanks so much for detailed explain. The minor plotting problem is solved.