esercizi marco
This commit is contained in:
Francesco Mecca
parent
31cb7db080
commit
8cdeb4b84b
23 changed files with 479534 additions and 1 deletions
|
|
@ -0,0 +1,365 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 81,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import math\n",
|
||||
"from copy import deepcopy\n",
|
||||
"import sklearn.datasets\n",
|
||||
"from sklearn.svm import SVC"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 64,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"X,y = sklearn.datasets.make_hastie_10_2()\n",
|
||||
"X_train = X[0:8000,:]\n",
|
||||
"y_train = y[0:8000]\n",
|
||||
"X_test = X[8000:,:]\n",
|
||||
"y_test = y[8000:]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 72,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class SVC_:\n",
|
||||
" def __init__(self, kernel=\"rbf\", degree=\"3\"):\n",
|
||||
" self.svc = SVC(kernel=kernel, degree=degree)\n",
|
||||
"\n",
|
||||
" def fit(self, X, y, sample_weight=None):\n",
|
||||
" if sample_weight is not None:\n",
|
||||
" sample_weight = sample_weight * len(X)\n",
|
||||
"\n",
|
||||
" self.svc.fit(X,y,sample_weight=sample_weight)\n",
|
||||
" return self\n",
|
||||
"\n",
|
||||
" def predict(self, X):\n",
|
||||
" return self.svc.predict(X)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Exercise 1\n",
|
||||
"\n",
|
||||
"1. Implement the AdaBoost ensemble algorithm by completing the following code:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 226,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class AdaBoost:\n",
|
||||
" def __init__(self, weakModel, T):\n",
|
||||
" self.model = weakModel\n",
|
||||
" self.models = []\n",
|
||||
" self.T = T\n",
|
||||
" self.a = []\n",
|
||||
"\n",
|
||||
" def fit(self, X, y):\n",
|
||||
" w = [1 / len(X) for x in X]\n",
|
||||
" for t in range(self.T):\n",
|
||||
" model = deepcopy(self.model)\n",
|
||||
" model.fit(X, y, sample_weight = w)\n",
|
||||
" predictions = model.predict(X)\n",
|
||||
" self.models.append(model)\n",
|
||||
" e = sum([w[i] if predictions[i] != y[i] else 0 for i in range(len(y))])\n",
|
||||
" if t%10 == 0:\n",
|
||||
" print(\"Weighted Error:\", e)\n",
|
||||
" a = np.log((1 - e) / e) / 2\n",
|
||||
" self.a.append(a)\n",
|
||||
" w = [w[i] * np.exp(-a * y[i] * predictions[i]) for i in range(len(w))]\n",
|
||||
" w /= sum(w)\n",
|
||||
" return self\n",
|
||||
"\n",
|
||||
" def predict(self, X):\n",
|
||||
" return np.sign(sum([self.a[t] * self.models[t].predict(X) for t in range(self.T)]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In the implementation you are free to assume:\n",
|
||||
"- that the problem is a binary classification problem with labels in $\\{-1, +1\\}$.\n",
|
||||
"- that the weakModel can fit a weighted sample set by means of the call `weakModel.fit(X,y,sample_weight=w)` where `w` is a vector of length $|y|$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"2. Test your implementation on the dataset loaded above and using an SVC with a polynomial kernel. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 227,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Weighted Error: 0.49512499999995935\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"weakModel = SVC(kernel=\"poly\", degree=3)\n",
|
||||
"adaboost = AdaBoost(weakModel, 10)\n",
|
||||
"adaboost.fit(X_train, y_train)\n",
|
||||
"y_test_ = adaboost.predict(X_test)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 123,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"0.49425\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(0.5 - (y_test.dot(y_test_)) / (2 * len(y_test)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"3. evaluate the AdaBoost performances as usual by calculating the classification error and compare it with the classification error of the weak model.\n",
|
||||
"\n",
|
||||
"**Note 1**: \n",
|
||||
"since the labels are bound to be in ${+1, -1}$, the classification error (i.e., the number of incorrectly classified examples over the total number of examples) can be easily computed as:\n",
|
||||
"$$\n",
|
||||
" error(y,y') = \\frac{N - y \\cdot y'}{2N} = \\frac{1}{2} - \\frac{y \\cdot y'}{2N},\n",
|
||||
"$$\n",
|
||||
"where $N$ is the total number of examples. The formula can be derived noticing that $y \\cdot y'$ calculates the number $N_c$ of examples correctly classified minus the number $N_{\\bar c}$ of examples incorrectly classified. We have then $y \\cdot y' = N_c - N_{\\bar c}$ and by noticing that $N = N_c + N_{\\bar c}$:\n",
|
||||
"$$\n",
|
||||
" N - y \\cdot y' = N_c + N_{\\bar c} - N_c + N_{\\bar c} = 2 N_{\\bar c} \\Rightarrow \\frac{N - y \\cdot y'}{2 N} = \\frac{N_{\\bar c}}{N}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"**Note 2**:\n",
|
||||
"do not forget to deepcopy your base model before fitting it to the new data\n",
|
||||
"\n",
|
||||
"**Note 3**:\n",
|
||||
"The SVC model allows specifying weights, but it *does not* work well when weights are normalized (it works well when the weights are larger). The following class takes normalized weights and denormalize them before passing them to the SVC classifier:\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
" class SVC_:\n",
|
||||
" def __init__(self, kernel=\"rbf\", degree=\"3\"):\n",
|
||||
" self.svc = SVC(kernel=kernel, degree=degree)\n",
|
||||
"\n",
|
||||
" def fit(self, X,y,sample_weight=None):\n",
|
||||
" if sample_weight is not None:\n",
|
||||
" sample_weight = sample_weight * len(X)\n",
|
||||
"\n",
|
||||
" self.svc.fit(X,y,sample_weight=sample_weight)\n",
|
||||
" return self\n",
|
||||
"\n",
|
||||
" def predict(self, X):\n",
|
||||
" return self.svc.predict(X)\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Exercise 2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"1. Write a weak learner to be used with the AdaBoost algorithm you just wrote. The weak learner that you will implement is the most inaccurate weak learner possible: it basically works by extracting a linear model at random and trying to use that model to classify the examples. Being extracted at random the models it generates do not guarantee that the weighted error $\\epsilon_t$ is smaller than $0.5$. The algorithm solves this problem by flipping the decisions whenever it finds out that $\\epsilon_t > 0.5$ (i.e., if the weighted error is larger than $0.5$ it reverses the sign of all the weights so that the decision surface stays the same, but the regions where it predicts $+1$ and $-1$ are reversed).\n",
|
||||
"\n",
|
||||
" It shall work as follows:\n",
|
||||
"\n",
|
||||
" - it creates a random linear model by generating the needed weight vector $\\mathbf{w}$ at random (**note**: these are the weights of the linear model, they are *NOT* related in any way to the weights of the examples); each weight shall be sampled from U(-1,1);\n",
|
||||
" - it evaluates the weighted loss $\\epsilon_t$ on the given dataset and flip the linear model if $\\epsilon_t > 0.5$;\n",
|
||||
" - at prediction time it predicts +1 if $\\mathbf{x} \\cdot \\mathbf{w} > 0$; it predicts -1 otherwise."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 222,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class RandomLinearModel:\n",
|
||||
" def loss(self, y, y_, sample_weight):\n",
|
||||
" return sum([sample_weight[i] if y[i] != y_[i] else 0 for i in range(len(y))])\n",
|
||||
" \n",
|
||||
" def fit(self,X,y,sample_weight=[]):\n",
|
||||
" self.w = np.random.rand(len(X[0])) * 2 - 1\n",
|
||||
" if len(sample_weight) == 0:\n",
|
||||
" sample_weight = [1 / len(X) for x in X]\n",
|
||||
" if self.loss(y, self.predict(X), sample_weight) > 0.5:\n",
|
||||
" self.w *= -1\n",
|
||||
" return self\n",
|
||||
" \n",
|
||||
" def predict(self,X):\n",
|
||||
" return np.sign(X.dot(self.w))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 228,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"0.487\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"rs = RandomLinearModel()\n",
|
||||
"rs.fit(X_train, y_train)\n",
|
||||
"predictions = rs.predict(X_test)\n",
|
||||
"print(0.5 - y_test.dot(predictions)/(2 * len(y_test)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"2. Learn an AdaBoost model using the RandomLinearModel weak learner printing every $K$ iterations the weighted error and the current error of the ensemble (you are free to choose $K$ so to make your output just frequent enough to let you know what is happening but without flooding the console with messages). Evaluate the training and test error of the final ensemble model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 229,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Weighted Error: 0.49524999999995933\n",
|
||||
"Weighted Error: 0.4948541341954795\n",
|
||||
"Weighted Error: 0.49729398392530305\n",
|
||||
"Weighted Error: 0.49980867302257964\n",
|
||||
"Weighted Error: 0.49683487146024025\n",
|
||||
"Weighted Error: 0.49790489175815233\n",
|
||||
"Weighted Error: 0.4940625587347454\n",
|
||||
"Weighted Error: 0.4950371378338745\n",
|
||||
"Weighted Error: 0.4909255291281916\n",
|
||||
"Weighted Error: 0.4960331784908466\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"rs = RandomLinearModel()\n",
|
||||
"a = AdaBoost(rs,100)\n",
|
||||
"a.fit(X_train,y_train)\n",
|
||||
"\n",
|
||||
"y_train_ = a.predict(X_train)\n",
|
||||
"y_test_ = a.predict(X_test)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 232,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Training Error: 0.462125\n",
|
||||
"Test Error: 0.49375\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\"Training Error:\", 0.5 - y_train.dot(y_train_)/(2 * len(y_train)))\n",
|
||||
"print(\"Test Error:\", 0.5 - y_test.dot(y_test_)/(2 * len(y_test)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"3. Write few paragraphs about what you think about the experiment and about the results you obtained."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"anaconda-cloud": {},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
385
anno3/apprendimento_automatico/esercizi/marco/AdaBoost.ipynb
Normal file
385
anno3/apprendimento_automatico/esercizi/marco/AdaBoost.ipynb
Normal file
|
|
@ -0,0 +1,385 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import math\n",
|
||||
"from copy import deepcopy\n",
|
||||
"import sklearn.datasets\n",
|
||||
"from sklearn.svm import SVC"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"X,y = sklearn.datasets.make_hastie_10_2()\n",
|
||||
"X_train = X[0:8000,:]\n",
|
||||
"y_train = y[0:8000]\n",
|
||||
"X_test = X[8000:,:]\n",
|
||||
"y_test = y[8000:]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class SVC_:\n",
|
||||
" def __init__(self, kernel=\"rbf\", degree=\"3\"):\n",
|
||||
" self.svc = SVC(kernel=kernel, degree=degree)\n",
|
||||
"\n",
|
||||
" def fit(self, X, y, sample_weight=None):\n",
|
||||
" if sample_weight is not None:\n",
|
||||
" sample_weight = sample_weight * len(X)\n",
|
||||
"\n",
|
||||
" self.svc.fit(X,y,sample_weight=sample_weight)\n",
|
||||
" return self\n",
|
||||
"\n",
|
||||
" def predict(self, X):\n",
|
||||
" return self.svc.predict(X)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Exercise 1\n",
|
||||
"\n",
|
||||
"1. Implement the AdaBoost ensemble algorithm by completing the following code:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class AdaBoost:\n",
|
||||
" def __init__(self, weakModel, T):\n",
|
||||
" self.model = weakModel\n",
|
||||
" self.models = []\n",
|
||||
" self.T = T\n",
|
||||
" self.a = []\n",
|
||||
"\n",
|
||||
" def fit(self, X, y):\n",
|
||||
" w = np.array([1 / len(X) for x in X])\n",
|
||||
" for t in range(self.T):\n",
|
||||
" model = deepcopy(self.model)\n",
|
||||
" model.fit(X, y, sample_weight = w)\n",
|
||||
" predictions = model.predict(X)\n",
|
||||
" self.models.append(model)\n",
|
||||
" e = sum([w[i] if predictions[i] != y[i] else 0 for i in range(len(y))])\n",
|
||||
" if t%10 == 0:\n",
|
||||
" print(\"Weighted Error:\", e)\n",
|
||||
" a = np.log((1 - e) / e) / 2\n",
|
||||
" self.a.append(a)\n",
|
||||
" w = np.array([w[i] * np.exp(-a * y[i] * predictions[i]) for i in range(len(w))])\n",
|
||||
" w /= sum(w)\n",
|
||||
" return self\n",
|
||||
"\n",
|
||||
" def predict(self, X):\n",
|
||||
" return np.sign(sum([self.a[t] * self.models[t].predict(X) for t in range(self.T)]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In the implementation you are free to assume:\n",
|
||||
"- that the problem is a binary classification problem with labels in $\\{-1, +1\\}$.\n",
|
||||
"- that the weakModel can fit a weighted sample set by means of the call `weakModel.fit(X,y,sample_weight=w)` where `w` is a vector of length $|y|$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"2. Test your implementation on the dataset loaded above and using an SVC with a polynomial kernel. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"8000"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"len(X_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Weighted Error: 0.3538749999999749\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n",
|
||||
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
|
||||
" \"avoid this warning.\", FutureWarning)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"weakModel = SVC_(kernel=\"poly\", degree=3)\n",
|
||||
"adaboost = AdaBoost(weakModel, 10)\n",
|
||||
"adaboost.fit(X_train, y_train)\n",
|
||||
"y_test_ = adaboost.predict(X_test)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"0.22525\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(0.5 - (y_test.dot(y_test_)) / (2 * len(y_test)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"3. evaluate the AdaBoost performances as usual by calculating the classification error and compare it with the classification error of the weak model.\n",
|
||||
"\n",
|
||||
"**Note 1**: \n",
|
||||
"since the labels are bound to be in ${+1, -1}$, the classification error (i.e., the number of incorrectly classified examples over the total number of examples) can be easily computed as:\n",
|
||||
"$$\n",
|
||||
" error(y,y') = \\frac{N - y \\cdot y'}{2N} = \\frac{1}{2} - \\frac{y \\cdot y'}{2N},\n",
|
||||
"$$\n",
|
||||
"where $N$ is the total number of examples. The formula can be derived noticing that $y \\cdot y'$ calculates the number $N_c$ of examples correctly classified minus the number $N_{\\bar c}$ of examples incorrectly classified. We have then $y \\cdot y' = N_c - N_{\\bar c}$ and by noticing that $N = N_c + N_{\\bar c}$:\n",
|
||||
"$$\n",
|
||||
" N - y \\cdot y' = N_c + N_{\\bar c} - N_c + N_{\\bar c} = 2 N_{\\bar c} \\Rightarrow \\frac{N - y \\cdot y'}{2 N} = \\frac{N_{\\bar c}}{N}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"**Note 2**:\n",
|
||||
"do not forget to deepcopy your base model before fitting it to the new data\n",
|
||||
"\n",
|
||||
"**Note 3**:\n",
|
||||
"The SVC model allows specifying weights, but it *does not* work well when weights are normalized (it works well when the weights are larger). The following class takes normalized weights and denormalize them before passing them to the SVC classifier:\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
" class SVC_:\n",
|
||||
" def __init__(self, kernel=\"rbf\", degree=\"3\"):\n",
|
||||
" self.svc = SVC(kernel=kernel, degree=degree)\n",
|
||||
"\n",
|
||||
" def fit(self, X,y,sample_weight=None):\n",
|
||||
" if sample_weight is not None:\n",
|
||||
" sample_weight = sample_weight * len(X)\n",
|
||||
"\n",
|
||||
" self.svc.fit(X,y,sample_weight=sample_weight)\n",
|
||||
" return self\n",
|
||||
"\n",
|
||||
" def predict(self, X):\n",
|
||||
" return self.svc.predict(X)\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Exercise 2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"1. Write a weak learner to be used with the AdaBoost algorithm you just wrote. The weak learner that you will implement is the most inaccurate weak learner possible: it basically works by extracting a linear model at random and trying to use that model to classify the examples. Being extracted at random the models it generates do not guarantee that the weighted error $\\epsilon_t$ is smaller than $0.5$. The algorithm solves this problem by flipping the decisions whenever it finds out that $\\epsilon_t > 0.5$ (i.e., if the weighted error is larger than $0.5$ it reverses the sign of all the weights so that the decision surface stays the same, but the regions where it predicts $+1$ and $-1$ are reversed).\n",
|
||||
"\n",
|
||||
" It shall work as follows:\n",
|
||||
"\n",
|
||||
" - it creates a random linear model by generating the needed weight vector $\\mathbf{w}$ at random (**note**: these are the weights of the linear model, they are *NOT* related in any way to the weights of the examples); each weight shall be sampled from U(-1,1);\n",
|
||||
" - it evaluates the weighted loss $\\epsilon_t$ on the given dataset and flip the linear model if $\\epsilon_t > 0.5$;\n",
|
||||
" - at prediction time it predicts +1 if $\\mathbf{x} \\cdot \\mathbf{w} > 0$; it predicts -1 otherwise."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 222,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class RandomLinearModel:\n",
|
||||
" def loss(self, y, y_, sample_weight):\n",
|
||||
" return sum([sample_weight[i] if y[i] != y_[i] else 0 for i in range(len(y))])\n",
|
||||
" \n",
|
||||
" def fit(self,X,y,sample_weight=[]):\n",
|
||||
" self.w = np.random.rand(len(X[0])) * 2 - 1\n",
|
||||
" if len(sample_weight) == 0:\n",
|
||||
" sample_weight = [1 / len(X) for x in X]\n",
|
||||
" if self.loss(y, self.predict(X), sample_weight) > 0.5:\n",
|
||||
" self.w *= -1\n",
|
||||
" return self\n",
|
||||
" \n",
|
||||
" def predict(self,X):\n",
|
||||
" return np.sign(X.dot(self.w))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 228,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"0.487\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"rs = RandomLinearModel()\n",
|
||||
"rs.fit(X_train, y_train)\n",
|
||||
"predictions = rs.predict(X_test)\n",
|
||||
"print(0.5 - y_test.dot(predictions)/(2 * len(y_test)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"2. Learn an AdaBoost model using the RandomLinearModel weak learner printing every $K$ iterations the weighted error and the current error of the ensemble (you are free to choose $K$ so to make your output just frequent enough to let you know what is happening but without flooding the console with messages). Evaluate the training and test error of the final ensemble model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 229,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Weighted Error: 0.49524999999995933\n",
|
||||
"Weighted Error: 0.4948541341954795\n",
|
||||
"Weighted Error: 0.49729398392530305\n",
|
||||
"Weighted Error: 0.49980867302257964\n",
|
||||
"Weighted Error: 0.49683487146024025\n",
|
||||
"Weighted Error: 0.49790489175815233\n",
|
||||
"Weighted Error: 0.4940625587347454\n",
|
||||
"Weighted Error: 0.4950371378338745\n",
|
||||
"Weighted Error: 0.4909255291281916\n",
|
||||
"Weighted Error: 0.4960331784908466\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"rs = RandomLinearModel()\n",
|
||||
"a = AdaBoost(rs,100)\n",
|
||||
"a.fit(X_train,y_train)\n",
|
||||
"\n",
|
||||
"y_train_ = a.predict(X_train)\n",
|
||||
"y_test_ = a.predict(X_test)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 232,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Training Error: 0.462125\n",
|
||||
"Test Error: 0.49375\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(\"Training Error:\", 0.5 - y_train.dot(y_train_)/(2 * len(y_train)))\n",
|
||||
"print(\"Test Error:\", 0.5 - y_test.dot(y_test_)/(2 * len(y_test)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"3. Write few paragraphs about what you think about the experiment and about the results you obtained."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"anaconda-cloud": {},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
150,2
|
||||
x,y
|
||||
10,10
|
||||
11,11
|
||||
12,12
|
||||
13,13
|
||||
14,14
|
||||
15,15
|
||||
16,16
|
||||
17,17
|
||||
18,18
|
||||
19,19
|
||||
20,20
|
||||
21,21
|
||||
22,22
|
||||
23,23
|
||||
24,24
|
||||
25,25
|
||||
26,26
|
||||
27,27
|
||||
28,28
|
||||
29,29
|
||||
30,30
|
||||
31,31
|
||||
32,32
|
||||
33,33
|
||||
34,34
|
||||
35,35
|
||||
36,36
|
||||
37,37
|
||||
38,38
|
||||
39,39
|
||||
40,40
|
||||
41,41
|
||||
42,42
|
||||
43,43
|
||||
44,44
|
||||
45,45
|
||||
46,46
|
||||
47,47
|
||||
48,48
|
||||
49,49
|
||||
50,50
|
||||
51,51
|
||||
52,52
|
||||
53,53
|
||||
54,54
|
||||
55,55
|
||||
56,56
|
||||
57,57
|
||||
58,58
|
||||
80,80
|
||||
81,81
|
||||
82,82
|
||||
83,83
|
||||
84,84
|
||||
85,85
|
||||
86,86
|
||||
87,87
|
||||
88,88
|
||||
89,89
|
||||
90,90
|
||||
91,91
|
||||
92,92
|
||||
93,93
|
||||
94,94
|
||||
95,95
|
||||
96,96
|
||||
97,97
|
||||
98,98
|
||||
99,99
|
||||
100,100
|
||||
101,101
|
||||
102,102
|
||||
103,103
|
||||
104,104
|
||||
105,105
|
||||
106,106
|
||||
107,107
|
||||
108,108
|
||||
109,109
|
||||
110,110
|
||||
111,111
|
||||
112,112
|
||||
113,113
|
||||
114,114
|
||||
115,115
|
||||
116,116
|
||||
117,117
|
||||
118,118
|
||||
119,119
|
||||
120,120
|
||||
121,121
|
||||
122,122
|
||||
123,123
|
||||
124,124
|
||||
125,125
|
||||
126,126
|
||||
127,127
|
||||
128,128
|
||||
129,129
|
||||
120,10
|
||||
121,11
|
||||
122,12
|
||||
123,13
|
||||
124,14
|
||||
125,15
|
||||
126,16
|
||||
127,17
|
||||
128,18
|
||||
129,19
|
||||
130,20
|
||||
131,21
|
||||
132,22
|
||||
133,23
|
||||
134,24
|
||||
135,25
|
||||
136,26
|
||||
137,27
|
||||
138,28
|
||||
139,29
|
||||
140,30
|
||||
141,31
|
||||
142,32
|
||||
143,33
|
||||
144,34
|
||||
145,35
|
||||
146,36
|
||||
147,37
|
||||
148,38
|
||||
149,39
|
||||
150,40
|
||||
151,41
|
||||
152,42
|
||||
153,43
|
||||
154,44
|
||||
155,45
|
||||
156,46
|
||||
157,47
|
||||
158,48
|
||||
159,49
|
||||
160,50
|
||||
161,51
|
||||
162,52
|
||||
163,53
|
||||
164,54
|
||||
165,55
|
||||
166,56
|
||||
167,57
|
||||
168,58
|
||||
169,59
|
||||
170,60
|
||||
|
86560
anno3/apprendimento_automatico/esercizi/marco/Datasets/CURE-complete.csv
Normal file
86560
anno3/apprendimento_automatico/esercizi/marco/Datasets/CURE-complete.csv
Normal file
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
BIN
anno3/apprendimento_automatico/esercizi/marco/aaut-merged.pdf
Normal file
BIN
anno3/apprendimento_automatico/esercizi/marco/aaut-merged.pdf
Normal file
Binary file not shown.
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1,840 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Experimenting with least squares and its variants"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"from sklearn import datasets\n",
|
||||
"from scipy.optimize import fmin_bfgs\n",
|
||||
"import numpy as np\n",
|
||||
"from numpy.linalg import norm\n",
|
||||
"from numpy.linalg import inv"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Data preparation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"boston = datasets.load_boston()\n",
|
||||
"data = np.array(boston.data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The boston dataset is one of the standard regression problems used to experiment with learning algorithms. Below you can find the dataset description"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
".. _boston_dataset:\n",
|
||||
"\n",
|
||||
"Boston house prices dataset\n",
|
||||
"---------------------------\n",
|
||||
"\n",
|
||||
"**Data Set Characteristics:** \n",
|
||||
"\n",
|
||||
" :Number of Instances: 506 \n",
|
||||
"\n",
|
||||
" :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.\n",
|
||||
"\n",
|
||||
" :Attribute Information (in order):\n",
|
||||
" - CRIM per capita crime rate by town\n",
|
||||
" - ZN proportion of residential land zoned for lots over 25,000 sq.ft.\n",
|
||||
" - INDUS proportion of non-retail business acres per town\n",
|
||||
" - CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)\n",
|
||||
" - NOX nitric oxides concentration (parts per 10 million)\n",
|
||||
" - RM average number of rooms per dwelling\n",
|
||||
" - AGE proportion of owner-occupied units built prior to 1940\n",
|
||||
" - DIS weighted distances to five Boston employment centres\n",
|
||||
" - RAD index of accessibility to radial highways\n",
|
||||
" - TAX full-value property-tax rate per $10,000\n",
|
||||
" - PTRATIO pupil-teacher ratio by town\n",
|
||||
" - B 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town\n",
|
||||
" - LSTAT % lower status of the population\n",
|
||||
" - MEDV Median value of owner-occupied homes in $1000's\n",
|
||||
"\n",
|
||||
" :Missing Attribute Values: None\n",
|
||||
"\n",
|
||||
" :Creator: Harrison, D. and Rubinfeld, D.L.\n",
|
||||
"\n",
|
||||
"This is a copy of UCI ML housing dataset.\n",
|
||||
"https://archive.ics.uci.edu/ml/machine-learning-databases/housing/\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.\n",
|
||||
"\n",
|
||||
"The Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic\n",
|
||||
"prices and the demand for clean air', J. Environ. Economics & Management,\n",
|
||||
"vol.5, 81-102, 1978. Used in Belsley, Kuh & Welsch, 'Regression diagnostics\n",
|
||||
"...', Wiley, 1980. N.B. Various transformations are used in the table on\n",
|
||||
"pages 244-261 of the latter.\n",
|
||||
"\n",
|
||||
"The Boston house-price data has been used in many machine learning papers that address regression\n",
|
||||
"problems. \n",
|
||||
" \n",
|
||||
".. topic:: References\n",
|
||||
"\n",
|
||||
" - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261.\n",
|
||||
" - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(boston.DESCR)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"First step to apply the formulae we learnt during the lectures is to rewrite the dataset in homogeneous coordinates (i.e., we append a column of 1 to the matrix containing the examples):"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"array([[1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.]])"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"t = np.ones(len(data)).reshape(len(data),1)\n",
|
||||
"data = np.append(data, t, 1)\n",
|
||||
"target = np.array(boston.target)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We now divide the data into a training set $X$ and a test set $X_\\textrm{test}$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"X,y = data[0:400,:], target[0:400]\n",
|
||||
"X_test, y_test = data[400:,:], target[400:]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Exercise"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"1. Calculate the least square solution (to the regression problem outlined above) and evaluate its performances on the training set and on the test set.\n",
|
||||
"1. Calculate the ridge regression solution (set lambda to 0.01) and evaluate its performances on the training set and on test set.\n",
|
||||
"1. Calculate the lasso regression solution and evaluate its performances on the training set and on the test set."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Notes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"- Here it follows a list of functions you may want to use (the required packages are already imported at the beginning of this notebook) along with a very brief explanation of their purpose (`help(nomefun)` will provide you more information about function `nomefun`):\n",
|
||||
" - `transpose`: matrix transposition (e.g., `transpose(X)`)\n",
|
||||
" - `dot`: matrix multiplication (e.g., `X.dot(X2)`) \n",
|
||||
" - `inv`: matrix inversion (e.g., `inv(X)`)\n",
|
||||
"- to solve the lasso problem you will need to perform a numerical minimization of the associated loss function (as you know, a closed form solution does not exist). There are many numerical optimization algorithms available in the scipy package. My suggestion is to use `fmin_bfgs`. Here it follows an example of how to use it:\n",
|
||||
" ```python\n",
|
||||
" def f(w):\n",
|
||||
" return w[0]**2 + w[1]**2 + w[0] + w[1]\n",
|
||||
" \n",
|
||||
" w = fmin_bfgs(f, [0,0])\n",
|
||||
" ```\n",
|
||||
" note that the function may (and should) reference your data variables (i.e., $X$ and $y$).\n",
|
||||
"- to evaluate the performances of your solutions use the $S$ statistic:\n",
|
||||
" $$\n",
|
||||
" S = \\sqrt{ \\frac{1}{n} \\sum_{i=1}^n (y_i' - y_i)^2 }\n",
|
||||
" $$\n",
|
||||
" where $y'_i$ is your model prediction for the i-th example, and $n$ is the number of examples."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 65,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import math"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 66,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def least_squares(X, y):\n",
|
||||
" return inv(np.transpose(X).dot(X)).dot(np.transpose(X)).dot(y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 67,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def ridge_regression(X, y, lam):\n",
|
||||
" return inv(np.transpose(X).dot(X) + np.identity(len(X[0])).dot(lam)).dot(np.transpose(X)).dot(y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 83,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def lasso(w, X, y, lam):\n",
|
||||
" return np.transpose(y - X.dot(w)).dot(y - X.dot(w)) + lam * sum(w)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 79,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def lasso_regression(X, y, lam):\n",
|
||||
" return fmin_bfgs(lasso, np.zeros(len(X[0])), args = (X, y, lam))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 70,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def s_statistics(actual, predicted):\n",
|
||||
" return math.sqrt(sum([(predicted[i] - actual[i])**2 for i in range(len(actual))]) / len(actual))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 80,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def predict(X, w):\n",
|
||||
" return X.dot(w)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 89,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Warning: Desired error not necessarily achieved due to precision loss.\n",
|
||||
" Current function value: 8922.270593\n",
|
||||
" Iterations: 19\n",
|
||||
" Function evaluations: 577\n",
|
||||
" Gradient evaluations: 36\n",
|
||||
"Least squares training set s statistics: 4.7228408383263805\n",
|
||||
"Least squares test set s statistics: 6.155792280414106\n",
|
||||
"Ridge regression training set s statistics: 4.7228952650983205\n",
|
||||
"Ridge regression test set s statistics: 6.141787930906379\n",
|
||||
"Lasso regression training set s statistics: 4.722840843957779\n",
|
||||
"Lasso regression test set s statistics: 6.1558291277697\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"w_least = least_squares(X, y)\n",
|
||||
"w_ridge = ridge_regression(X, y, 0.01)\n",
|
||||
"w_lasso = lasso_regression(X, y, 0.01)\n",
|
||||
"print(\"Least squares training set s statistics:\", s_statistics(y, predict(X, w_least)))\n",
|
||||
"print(\"Least squares test set s statistics:\", s_statistics(y_test, predict(X_test, w_least)))\n",
|
||||
"print(\"Ridge regression training set s statistics:\", s_statistics(y, predict(X, w_ridge)))\n",
|
||||
"print(\"Ridge regression test set s statistics:\", s_statistics(y_test, predict(X_test, w_ridge)))\n",
|
||||
"print(\"Lasso regression training set s statistics:\", s_statistics(y, predict(X, w_lasso)))\n",
|
||||
"print(\"Lasso regression test set s statistics:\", s_statistics(y_test, predict(X_test, w_lasso)))"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"anaconda-cloud": {},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
|
|
@ -0,0 +1,840 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Experimenting with least squares and its variants"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"from sklearn import datasets\n",
|
||||
"from scipy.optimize import fmin_bfgs\n",
|
||||
"import numpy as np\n",
|
||||
"from numpy.linalg import norm\n",
|
||||
"from numpy.linalg import inv"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Data preparation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"boston = datasets.load_boston()\n",
|
||||
"data = np.array(boston.data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The boston dataset is one of the standard regression problems used to experiment with learning algorithms. Below you can find the dataset description"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
".. _boston_dataset:\n",
|
||||
"\n",
|
||||
"Boston house prices dataset\n",
|
||||
"---------------------------\n",
|
||||
"\n",
|
||||
"**Data Set Characteristics:** \n",
|
||||
"\n",
|
||||
" :Number of Instances: 506 \n",
|
||||
"\n",
|
||||
" :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.\n",
|
||||
"\n",
|
||||
" :Attribute Information (in order):\n",
|
||||
" - CRIM per capita crime rate by town\n",
|
||||
" - ZN proportion of residential land zoned for lots over 25,000 sq.ft.\n",
|
||||
" - INDUS proportion of non-retail business acres per town\n",
|
||||
" - CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)\n",
|
||||
" - NOX nitric oxides concentration (parts per 10 million)\n",
|
||||
" - RM average number of rooms per dwelling\n",
|
||||
" - AGE proportion of owner-occupied units built prior to 1940\n",
|
||||
" - DIS weighted distances to five Boston employment centres\n",
|
||||
" - RAD index of accessibility to radial highways\n",
|
||||
" - TAX full-value property-tax rate per $10,000\n",
|
||||
" - PTRATIO pupil-teacher ratio by town\n",
|
||||
" - B 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town\n",
|
||||
" - LSTAT % lower status of the population\n",
|
||||
" - MEDV Median value of owner-occupied homes in $1000's\n",
|
||||
"\n",
|
||||
" :Missing Attribute Values: None\n",
|
||||
"\n",
|
||||
" :Creator: Harrison, D. and Rubinfeld, D.L.\n",
|
||||
"\n",
|
||||
"This is a copy of UCI ML housing dataset.\n",
|
||||
"https://archive.ics.uci.edu/ml/machine-learning-databases/housing/\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.\n",
|
||||
"\n",
|
||||
"The Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic\n",
|
||||
"prices and the demand for clean air', J. Environ. Economics & Management,\n",
|
||||
"vol.5, 81-102, 1978. Used in Belsley, Kuh & Welsch, 'Regression diagnostics\n",
|
||||
"...', Wiley, 1980. N.B. Various transformations are used in the table on\n",
|
||||
"pages 244-261 of the latter.\n",
|
||||
"\n",
|
||||
"The Boston house-price data has been used in many machine learning papers that address regression\n",
|
||||
"problems. \n",
|
||||
" \n",
|
||||
".. topic:: References\n",
|
||||
"\n",
|
||||
" - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261.\n",
|
||||
" - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(boston.DESCR)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"First step to apply the formulae we learnt during the lectures is to rewrite the dataset in homogeneous coordinates (i.e., we append a column of 1 to the matrix containing the examples):"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"array([[1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.],\n",
|
||||
" [1.]])"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"t = np.ones(len(data)).reshape(len(data),1)\n",
|
||||
"data = np.append(data, t, 1)\n",
|
||||
"target = np.array(boston.target)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We now divide the data into a training set $X$ and a test set $X_\\textrm{test}$."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"X,y = data[0:400,:], target[0:400]\n",
|
||||
"X_test, y_test = data[400:,:], target[400:]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Exercise"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"1. Calculate the least square solution (to the regression problem outlined above) and evaluate its performances on the training set and on the test set.\n",
|
||||
"1. Calculate the ridge regression solution (set lambda to 0.01) and evaluate its performances on the training set and on test set.\n",
|
||||
"1. Calculate the lasso regression solution and evaluate its performances on the training set and on the test set."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Notes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"- Here it follows a list of functions you may want to use (the required packages are already imported at the beginning of this notebook) along with a very brief explanation of their purpose (`help(nomefun)` will provide you more information about function `nomefun`):\n",
|
||||
" - `transpose`: matrix transposition (e.g., `transpose(X)`)\n",
|
||||
" - `dot`: matrix multiplication (e.g., `X.dot(X2)`) \n",
|
||||
" - `inv`: matrix inversion (e.g., `inv(X)`)\n",
|
||||
"- to solve the lasso problem you will need to perform a numerical minimization of the associated loss function (as you know, a closed form solution does not exist). There are many numerical optimization algorithms available in the scipy package. My suggestion is to use `fmin_bfgs`. Here it follows an example of how to use it:\n",
|
||||
" ```python\n",
|
||||
" def f(w):\n",
|
||||
" return w[0]**2 + w[1]**2 + w[0] + w[1]\n",
|
||||
" \n",
|
||||
" w = fmin_bfgs(f, [0,0])\n",
|
||||
" ```\n",
|
||||
" note that the function may (and should) reference your data variables (i.e., $X$ and $y$).\n",
|
||||
"- to evaluate the performances of your solutions use the $S$ statistic:\n",
|
||||
" $$\n",
|
||||
" S = \\sqrt{ \\frac{1}{n} \\sum_{i=1}^n (y_i' - y_i)^2 }\n",
|
||||
" $$\n",
|
||||
" where $y'_i$ is your model prediction for the i-th example, and $n$ is the number of examples."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 65,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import math"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 66,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def least_squares(X, y):\n",
|
||||
" return inv(np.transpose(X).dot(X)).dot(np.transpose(X)).dot(y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 67,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def ridge_regression(X, y, lam):\n",
|
||||
" return inv(np.transpose(X).dot(X) + np.identity(len(X[0])).dot(lam)).dot(np.transpose(X)).dot(y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 83,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def lasso(w, X, y, lam):\n",
|
||||
" return np.transpose(y - X.dot(w)).dot(y - X.dot(w)) + lam * sum(w)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 79,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def lasso_regression(X, y, lam):\n",
|
||||
" return fmin_bfgs(lasso, np.zeros(len(X[0])), args = (X, y, lam))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 70,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def s_statistics(actual, predicted):\n",
|
||||
" return math.sqrt(sum([(predicted[i] - actual[i])**2 for i in range(len(actual))]) / len(actual))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 80,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def predict(X, w):\n",
|
||||
" return X.dot(w)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 89,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Warning: Desired error not necessarily achieved due to precision loss.\n",
|
||||
" Current function value: 8922.270593\n",
|
||||
" Iterations: 19\n",
|
||||
" Function evaluations: 577\n",
|
||||
" Gradient evaluations: 36\n",
|
||||
"Least squares training set s statistics: 4.7228408383263805\n",
|
||||
"Least squares test set s statistics: 6.155792280414106\n",
|
||||
"Ridge regression training set s statistics: 4.7228952650983205\n",
|
||||
"Ridge regression test set s statistics: 6.141787930906379\n",
|
||||
"Lasso regression training set s statistics: 4.722840843957779\n",
|
||||
"Lasso regression test set s statistics: 6.1558291277697\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"w_least = least_squares(X, y)\n",
|
||||
"w_ridge = ridge_regression(X, y, 0.01)\n",
|
||||
"w_lasso = lasso_regression(X, y, 0.01)\n",
|
||||
"print(\"Least squares training set s statistics:\", s_statistics(y, predict(X, w_least)))\n",
|
||||
"print(\"Least squares test set s statistics:\", s_statistics(y_test, predict(X_test, w_least)))\n",
|
||||
"print(\"Ridge regression training set s statistics:\", s_statistics(y, predict(X, w_ridge)))\n",
|
||||
"print(\"Ridge regression test set s statistics:\", s_statistics(y_test, predict(X_test, w_ridge)))\n",
|
||||
"print(\"Lasso regression training set s statistics:\", s_statistics(y, predict(X, w_lasso)))\n",
|
||||
"print(\"Lasso regression test set s statistics:\", s_statistics(y_test, predict(X_test, w_lasso)))"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"anaconda-cloud": {},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 1
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
digraph Tree {
|
||||
node [shape=box] ;
|
||||
0 [label="X[2] <= 2.45\nentropy = 1.585\nsamples = 150\nvalue = [50, 50, 50]"] ;
|
||||
1 [label="entropy = 0.0\nsamples = 50\nvalue = [50, 0, 0]"] ;
|
||||
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
|
||||
2 [label="X[3] <= 1.75\nentropy = 1.0\nsamples = 100\nvalue = [0, 50, 50]"] ;
|
||||
0 -> 2 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
|
||||
3 [label="X[2] <= 4.95\nentropy = 0.445\nsamples = 54\nvalue = [0, 49, 5]"] ;
|
||||
2 -> 3 ;
|
||||
4 [label="X[0] <= 5.15\nentropy = 0.146\nsamples = 48\nvalue = [0, 47, 1]"] ;
|
||||
3 -> 4 ;
|
||||
5 [label="entropy = 0.722\nsamples = 5\nvalue = [0, 4, 1]"] ;
|
||||
4 -> 5 ;
|
||||
6 [label="entropy = 0.0\nsamples = 43\nvalue = [0, 43, 0]"] ;
|
||||
4 -> 6 ;
|
||||
7 [label="entropy = 0.918\nsamples = 6\nvalue = [0, 2, 4]"] ;
|
||||
3 -> 7 ;
|
||||
8 [label="X[2] <= 4.95\nentropy = 0.151\nsamples = 46\nvalue = [0, 1, 45]"] ;
|
||||
2 -> 8 ;
|
||||
9 [label="entropy = 0.65\nsamples = 6\nvalue = [0, 1, 5]"] ;
|
||||
8 -> 9 ;
|
||||
10 [label="entropy = 0.0\nsamples = 40\nvalue = [0, 0, 40]"] ;
|
||||
8 -> 10 ;
|
||||
}
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
324
anno3/apprendimento_automatico/esercizi/marco/one_vs_rest.ipynb
Normal file
324
anno3/apprendimento_automatico/esercizi/marco/one_vs_rest.ipynb
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
617
anno3/apprendimento_automatico/esercizi/marco/svm.ipynb
Normal file
617
anno3/apprendimento_automatico/esercizi/marco/svm.ipynb
Normal file
File diff suppressed because one or more lines are too long
1
media.py
1
media.py
|
|
@ -10,6 +10,7 @@ voti = [
|
|||
(30, 6), # mcad
|
||||
(30, 6), # scpd
|
||||
(27, 9), # vpc
|
||||
(30, 6), # progmobile
|
||||
]
|
||||
|
||||
crediti, voto = sum(map(lambda x: x[1], voti)), sum(map(lambda x: x[0]*x[1], voti))
|
||||
|
|
|
|||
Loading…
Reference in a new issue