山花落尽，我心犹青

1
import hashlib
2
import os
3
import tarfile
4
import zipfile
5
import requests
6

7
DATA_HUB = dict()
8
DATA_URL = 'http://d2l-data.s3-accelerate.amazonaws.com/'
9

10
def download(name, cache_dir=os.path.join('..', 'data')):
11
    """下载数据集并校验SHA-1哈希值，支持缓存以避免重复下载。"""
12
    assert name in DATA_HUB, f"{name}不在{DATA_HUB}中"
13
    url, sha1_hash = DATA_HUB[name]
14
    os.makedirs(cache_dir, exist_ok=True)
15
    fname = os.path.join(cache_dir, url.split('/')[-1])
16
    # 检查文件是否已存在且哈希值匹配
17
    if os.path.exists(fname):
18
        sha1 = hashlib.sha1()
19
        with open(fname, 'rb') as f:
20
            while True:
21
                data = f.read(1048576) # 读取1MB数据块
22
                if not data:
23
                    break
24
                sha1.update(data)
25
        if sha1.hexdigest() == sha1_hash:
26
            return fname # 缓存有效，直接返回路径
27
    # 下载文件
28
    print(f"正在从{url}下载{fname}")
29
    r = requests.get(url, stream=True, verify=True)
30
    with open(fname, 'wb') as f:
31
        f.write(r.content)
32
    return fname

1
def download_extract(name, folder=None):
2
    """下载并解压文件。"""
3
    fname = download(name)
4
    base_dir = os.path.dirname(fname)
5
    data_dir, ext = os.path.splitext(fname)
6
    if ext == '.zip':
7
        fp = zipfile.ZipFile(fname, 'r')
8
    elif ext in ('.tar', '.gz'):
9
        fp = tarfile.open(fname, 'r')
10
    else:
11
        assert False, '文件类型不受支持'
12
    fp.extractall(base_dir)
13
    return os.path.join(base_dir, folder) if folder else data_dir

1
# 注册数据集URL和哈希值
2
DATA_HUB['kaggle_house_train'] = (
3
    DATA_URL + 'kaggle_house_pred_train.csv',
4
    '585e9cc93e70b39160e7921475f9bcd7d31219ce')
5
DATA_HUB['kaggle_house_test'] = (
6
    DATA_URL + 'kaggle_house_pred_test.csv',
7
    'fa19780a7b011d9b009e8bff8e99922a8ee2eb90')
8

9
# 使用pandas读取CSV数据
10
import pandas as pd
11
train_data = pd.read_csv(download('kaggle_house_train'))
12
test_data = pd.read_csv(download('kaggle_house_test'))
13

14
print(f'训练集形状: {train_data.shape}') # 输出: (1460, 81)
15
print(f'测试集形状: {test_data.shape}')   # 输出: (1459, 80)

1
# 移除训练集中的‘Id’列和‘SalePrice’标签列，移除测试集中的‘Id’列
2
all_features = pd.concat((train_data.iloc[:, 1:-1], test_data.iloc[:, 1:]))

1
numeric_features = all_features.dtypes[all_features.dtypes != 'object'].index

1
all_features[numeric_features] = all_features[numeric_features].apply(
2
    lambda x: (x - x.mean()) / (x.std()))

1
all_features[numeric_features] = all_features[numeric_features].fillna(0)

1
all_features = pd.get_dummies(all_features, dummy_na=True)
2
print(f'编码后特征维度: {all_features.shape}')

1
all_features = all_features.replace({True: 1, False: 0})

1
import torch
2

3
n_train = train_data.shape[0]
4
# 分割回训练集和测试集
5
train_features = torch.tensor(all_features[:n_train].values, dtype=torch.float32)
6
test_features = torch.tensor(all_features[n_train:].values, dtype=torch.float32)
7
# 提取训练标签（房价）
8
train_labels = torch.tensor(
9
    train_data.SalePrice.values.reshape(-1, 1), dtype=torch.float32)

1
from torch import nn
2

3
in_features = train_features.shape[1]
4
loss = nn.MSELoss() # 均方误差损失，适用于回归问题
5

6
def get_net():
7
    """定义一个简单的单层线性回归网络。"""
8
    net = nn.Sequential(nn.Linear(in_features, 1))
9
    return net

1
def log_rmse(net, features, labels):
2
    clipped_preds = torch.clamp(net(features), 1, float('inf'))
3
    rmse = torch.sqrt(loss(torch.log(clipped_preds), torch.log(labels)))
4
    return rmse.item()

1
def train(net, train_features, train_labels, test_features, test_labels,
2
          num_epochs, learning_rate, weight_decay, batch_size):
3
    train_ls, test_ls = [], []
4
    # 使用d2l库中的工具函数创建数据迭代器
5
    train_iter = d2l.load_array((train_features, train_labels), batch_size)
6
    optimizer = torch.optim.Adam(net.parameters(),
7
                                 lr=learning_rate,
8
                                 weight_decay=weight_decay)
9
    for epoch in range(num_epochs):
10
        for X, y in train_iter:
11
            optimizer.zero_grad()
12
            l = loss(net(X), y)
13
            l.backward()
14
            optimizer.step()
15
        train_ls.append(log_rmse(net, train_features, train_labels))
16
        if test_labels is not None:
17
            test_ls.append(log_rmse(net, test_features, test_labels))
18
    return train_ls, test_ls

1
def get_k_fold_data(k, i, X, y):
2
    """获取第i折交叉验证所需的数据。"""
3
    assert k > 1
4
    fold_size = X.shape[0] // k
5
    X_train, y_train = None, None
6
    for j in range(k):
7
        idx = slice(j * fold_size, (j + 1) * fold_size)
8
        X_part, y_part = X[idx, :], y[idx]
9
        if j == i: # 第i份作为验证集
10
            X_valid, y_valid = X_part, y_part
11
        elif X_train is None:
12
            X_train, y_train = X_part, y_part
13
        else:
14
            X_train = torch.cat([X_train, X_part], 0)
15
            y_train = torch.cat([y_train, y_part], 0)
16
    return X_train, y_train, X_valid, y_valid
17

18
def k_fold(k, X_train, y_train, num_epochs, lr, weight_decay, batch_size):
19
    """执行K折交叉验证，返回平均训练和验证误差。"""
20
    train_l_sum, valid_l_sum = 0, 0
21
    for i in range(k):
22
        data = get_k_fold_data(k, i, X_train, y_train)
23
        net = get_net()
24
        train_ls, valid_ls = train(net, *data, num_epochs, lr,
25
                                   weight_decay, batch_size)
26
        train_l_sum += train_ls[-1]
27
        valid_l_sum += valid_ls[-1]
28
        print(f'折{i + 1}，训练log rmse: {float(train_ls[-1]):f}, '
29
              f'验证log rmse: {float(valid_l[-1]):f}')
30
    return train_l_sum / k, valid_l_sum / k

1
k, num_epochs, lr, weight_decay, batch_size = 5, 100, 5, 0, 64
2
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
3
                          weight_decay, batch_size)
4
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
5
      f'平均验证log rmse: {float(valid_l):f}')