。这三个方法都需要将数据集分为测试集和训练集，用测试集的“测试误差”来近似模型的泛化误差。当然测试集和训练集应该尽可能做到互斥，这样得到的模型才具有较好的泛化性能。

1.留出法

#!/usr/bin/env python # -*- coding: utf-8 -*- from sklearn.model_selection
import StratifiedShuffleSplit import numpy as np X = np.array([[0, 1], [0, 2],
[0, 3], [1, 8], [1,9], [1,10]]) y = np.array([0, 0, 0, 1, 1, 1])
#测试集的比例为0.333，采用分层抽样。且随机进行5次 sss = StratifiedShuffleSplit(n_splits=5,
test_size=0.333) for train_index, test_index in sss.split(X, y):
print("TRAIN:", train_index, "TEST:", test_index) X_train, X_test =
X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index]

2.交叉验证法

cross validation）。常用的有10折交叉验证方法，特殊的有留一法（Leave- One-Out cross validation, LOOCV）。

（1）10折交叉验证

#!/usr/bin/env python # -*- coding: utf-8 -*- import loadData import copy
import numpy as np from sklearn import svm from sklearn.model_selection import
StratifiedKFold from sklearn.metrics import accuracy_score feature ,label =
foldsList = [] ss = StratifiedKFold(n_splits=10, shuffle = True) for
train_index, test_index in ss.split(feature, label): print("TEST:",
test_index)#获得索引值 foldsList.append(test_index) test_accur_list = [] for i in
xrange(10):#外层循环，9份训练，一份测试 train_index = list(set(range(0, smps)) -
set(foldsList[i])) test_index = foldsList[i] train_fea, test_fea =
feature[train_index], feature[test_index] train_lab, test_lab =
label[train_index], label[test_index] foldLi = copy.deepcopy(foldsList)
#删除测试的那一份 del foldLi[i] #将foldLi里面是list合并 foldL = [x for p in foldLi for x in
p] print 'for %s time gridSearch process:' % i c_can = np.logspace(-15, 15, 10,
base = 2)#假定SVM中的参数C的取值 n_search = len(c_can) bestC = 0
#对于这次的9份训练数据，通过下面内层交叉验证确定的最佳C值 bestAccur = 0 #内层交叉验证的最佳测试精度，即C取bestC时 for j in
xrange(n_search):#对参数C挨个进行测试 Accur = 0 for n in xrange(9):#C取值为c_can[j]时的内层交叉验证
train_i = list(set(foldL) - set(foldLi[n])) test_i= foldLi[n] train_f, test_f =
feature[train_i], feature[test_i]#训练集对应的值 train_l, test_l = label[train_i],
label[test_i]#类别集对应的值 clf = svm.SVC(C = c_can[j], kernel='linear')
clf.fit(train_f, train_l) y_hat = clf.predict(test_f) Accur +=
accuracy_score(test_l, y_hat) / 9 print ' Accur:%s' % Accur if Accur >
bestAccur:#根据内层交叉验证结果，查找bestC bestAccur = copy.deepcopy(Accur) bestC =
copy.deepcopy(c_can[j]) print ' Best validation accuracy on current dataset
split:', bestAccur print ' Best para C:', bestC #找到bestC后，然后才是外层的一次：训练、测试。 clf
= svm.SVC(C = bestC, kernel='linear') clf.fit(train_fea, train_lab) y_hat =
clf.predict(test_fea) test_accur_list.append(accuracy_score(test_lab, y_hat))
print ' test accur:', test_accur_list[i] print '\n' #最终得到十折交叉验证的结果 print
'average test accur:', sum(test_accur_list) / len(test_accur_list)

（2）留一法(LOOCV）

#!/usr/bin/env python # -*- coding: utf-8 -*- import loadData import copy
import numpy as np from sklearn import svm from sklearn.model_selection import
StratifiedKFold from sklearn.metrics import accuracy_score feature ,label =
for i in xrange(smps):#外层循环，LOOCV:一个样本测试，剩余训练 train_index = list(set(range(0,
smps)) - set([i])) test_index = [i] train_fea, test_fea = feature[train_index],
feature[test_index] train_lab, test_lab = label[train_index], label[test_index]
foldLi = [] ss = StratifiedKFold(n_splits=10, shuffle = True) for train_i,
test_i in ss.split(train_fea, train_lab): print("TEST:", test_i)#获得索引值
foldLi.append(test_i) #将foldLi里面是list合并 foldL = [x for p in foldLi for x in p]
print 'for %s time gridSearch process:' % i c_can = np.logspace(-15, 15, 10,
base = 2)#假定SVM中的参数C的取值 n_search = len(c_can) bestC = 0
#对于这次的9份训练数据，通过下面内层交叉验证确定的最佳C值 bestAccur = 0 #内层交叉验证的最佳测试精度，即C取bestC时 for j in
xrange(n_search):#对参数C挨个进行测试 Accur = 0 for n in
xrange(10):#C取值为c_can[j]时的内层交叉验证 train_i = list(set(foldL) - set(foldLi[n]))
test_i= foldLi[n] train_f, test_f = feature[train_i], feature[test_i]#训练集对应的值
train_l, test_l = label[train_i], label[test_i]#类别集对应的值 clf = svm.SVC(C =
c_can[j], kernel='linear') clf.fit(train_f, train_l) y_hat =
clf.predict(test_f) Accur += accuracy_score(test_l, y_hat) / 10 print '
Accur:%s' % Accur if Accur > bestAccur:#根据内层交叉验证结果，查找bestC bestAccur =
copy.deepcopy(Accur) bestC = copy.deepcopy(c_can[j]) print ' Best validation
accuracy on current dataset split:', bestAccur print ' Best para C:', bestC
#找到bestC后，然后才是外层的一次：训练、测试。 clf = svm.SVC(C = bestC, kernel='linear')
clf.fit(train_fea, train_lab) y_hat = clf.predict(test_fea)
test_accur_list.append(accuracy_score(test_lab, y_hat)) print ' test accur:',
test_accur_list[i] print '\n' #LOOCV的结果 print 'average test accur:',
sum(test_accur_list) / len(test_accur_list)
3.自助法

#!/usr/bin/env python # -*- coding: utf-8 -*- import loadData import copy
import numpy as np from sklearn import svm from sklearn.model_selection import
StratifiedKFold from sklearn.metrics import accuracy_score feature ,label =
#使用自助采样法，重复执行10次 for i in xrange(10): train_index = [] #自助法抽样,随机有放回地抽取samps个样本
for i in xrange(int(1.0 * samps)): train_index.append(random.randint(0,samps -
1)) test_index = list(set(list(range(0, samps))) - set(train_index))
print("TRAIN:", train_index, "TEST:", test_index)#获得索引值 train_fea, test_fea =
feature[train_index], feature[test_index]#训练集对应的值 train_lab, test_lab =
label[train_index], label[test_index]#类别集对应的值 ''' ''' 利用训练样本进行交叉验证，获取参数:C =
bestC ''' ''' clf = svm.SVC(C = bestC, kernel='linear') clf.fit(train_fea,
train_lab) y_hat = clf.predict(test_fea) accur = accuracy_score(test_lab,
y_hat) print i, ' time run testAccuracy:', accur testAccur_list.append(accur)
print '10 times run testAccur_list:', testAccur_list print 'average testAccur
value:', sum(testAccur_list) / 10

1.《机器学习》，周志华。

2.Statnikov A, Aliferis C F, Tsamardinos I, et al. A comprehensive evaluation
of multicategory classification methods for microarray gene expression cancer
diagnosis[J]. Bioinformatics, 2004, 21(5): 631-643.