bparodi/UniTO
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

esercizi marco

Browse source
This commit is contained in:
Francesco Mecca 2020-06-17 20:01:41 +02:00
parent 31cb7db080
commit 8cdeb4b84b
23 changed files with 479534 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

365
anno3/apprendimento_automatico/esercizi/marco/AdaBoost-checkpoint.ipynb Normal file
View file

@ -0,0 +1,365 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 81,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import math\n",
"from copy import deepcopy\n",
"import sklearn.datasets\n",
"from sklearn.svm import SVC"
]
},
{
"cell_type": "code",
"execution_count": 64,
"metadata": {},
"outputs": [],
"source": [
"X,y = sklearn.datasets.make_hastie_10_2()\n",
"X_train = X[0:8000,:]\n",
"y_train = y[0:8000]\n",
"X_test = X[8000:,:]\n",
"y_test = y[8000:]"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {},
"outputs": [],
"source": [
"class SVC_:\n",
" def __init__(self, kernel=\"rbf\", degree=\"3\"):\n",
" self.svc = SVC(kernel=kernel, degree=degree)\n",
"\n",
" def fit(self, X, y, sample_weight=None):\n",
" if sample_weight is not None:\n",
" sample_weight = sample_weight * len(X)\n",
"\n",
" self.svc.fit(X,y,sample_weight=sample_weight)\n",
" return self\n",
"\n",
" def predict(self, X):\n",
" return self.svc.predict(X)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Exercise 1\n",
"\n",
"1. Implement the AdaBoost ensemble algorithm by completing the following code:"
]
},
{
"cell_type": "code",
"execution_count": 226,
"metadata": {},
"outputs": [],
"source": [
"class AdaBoost:\n",
" def __init__(self, weakModel, T):\n",
" self.model = weakModel\n",
" self.models = []\n",
" self.T = T\n",
" self.a = []\n",
"\n",
" def fit(self, X, y):\n",
" w = [1 / len(X) for x in X]\n",
" for t in range(self.T):\n",
" model = deepcopy(self.model)\n",
" model.fit(X, y, sample_weight = w)\n",
" predictions = model.predict(X)\n",
" self.models.append(model)\n",
" e = sum([w[i] if predictions[i] != y[i] else 0 for i in range(len(y))])\n",
" if t%10 == 0:\n",
" print(\"Weighted Error:\", e)\n",
" a = np.log((1 - e) / e) / 2\n",
" self.a.append(a)\n",
" w = [w[i] * np.exp(-a * y[i] * predictions[i]) for i in range(len(w))]\n",
" w /= sum(w)\n",
" return self\n",
"\n",
" def predict(self, X):\n",
" return np.sign(sum([self.a[t] * self.models[t].predict(X) for t in range(self.T)]))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In the implementation you are free to assume:\n",
"- that the problem is a binary classification problem with labels in $\\{-1, +1\\}$.\n",
"- that the weakModel can fit a weighted sample set by means of the call `weakModel.fit(X,y,sample_weight=w)` where `w` is a vector of length $|y|$."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"2. Test your implementation on the dataset loaded above and using an SVC with a polynomial kernel. "
]
},
{
"cell_type": "code",
"execution_count": 227,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Weighted Error: 0.49512499999995935\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n"
]
}
],
"source": [
"weakModel = SVC(kernel=\"poly\", degree=3)\n",
"adaboost = AdaBoost(weakModel, 10)\n",
"adaboost.fit(X_train, y_train)\n",
"y_test_ = adaboost.predict(X_test)"
]
},
{
"cell_type": "code",
"execution_count": 123,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.49425\n"
]
}
],
"source": [
"print(0.5 - (y_test.dot(y_test_)) / (2 * len(y_test)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"3. evaluate the AdaBoost performances as usual by calculating the classification error and compare it with the classification error of the weak model.\n",
"\n",
"**Note 1**: \n",
"since the labels are bound to be in ${+1, -1}$, the classification error (i.e., the number of incorrectly classified examples over the total number of examples) can be easily computed as:\n",
"$$\n",
" error(y,y') = \\frac{N - y \\cdot y'}{2N} = \\frac{1}{2} - \\frac{y \\cdot y'}{2N},\n",
"$$\n",
"where $N$ is the total number of examples. The formula can be derived noticing that $y \\cdot y'$ calculates the number $N_c$ of examples correctly classified minus the number $N_{\\bar c}$ of examples incorrectly classified. We have then $y \\cdot y' = N_c - N_{\\bar c}$ and by noticing that $N = N_c + N_{\\bar c}$:\n",
"$$\n",
" N - y \\cdot y' = N_c + N_{\\bar c} - N_c + N_{\\bar c} = 2 N_{\\bar c} \\Rightarrow \\frac{N - y \\cdot y'}{2 N} = \\frac{N_{\\bar c}}{N}\n",
"$$\n",
"\n",
"**Note 2**:\n",
"do not forget to deepcopy your base model before fitting it to the new data\n",
"\n",
"**Note 3**:\n",
"The SVC model allows specifying weights, but it *does not* work well when weights are normalized (it works well when the weights are larger). The following class takes normalized weights and denormalize them before passing them to the SVC classifier:\n",
"\n",
"```python\n",
" class SVC_:\n",
" def __init__(self, kernel=\"rbf\", degree=\"3\"):\n",
" self.svc = SVC(kernel=kernel, degree=degree)\n",
"\n",
" def fit(self, X,y,sample_weight=None):\n",
" if sample_weight is not None:\n",
" sample_weight = sample_weight * len(X)\n",
"\n",
" self.svc.fit(X,y,sample_weight=sample_weight)\n",
" return self\n",
"\n",
" def predict(self, X):\n",
" return self.svc.predict(X)\n",
"```"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Exercise 2"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"1. Write a weak learner to be used with the AdaBoost algorithm you just wrote. The weak learner that you will implement is the most inaccurate weak learner possible: it basically works by extracting a linear model at random and trying to use that model to classify the examples. Being extracted at random the models it generates do not guarantee that the weighted error $\\epsilon_t$ is smaller than $0.5$. The algorithm solves this problem by flipping the decisions whenever it finds out that $\\epsilon_t > 0.5$ (i.e., if the weighted error is larger than $0.5$ it reverses the sign of all the weights so that the decision surface stays the same, but the regions where it predicts $+1$ and $-1$ are reversed).\n",
"\n",
" It shall work as follows:\n",
"\n",
" - it creates a random linear model by generating the needed weight vector $\\mathbf{w}$ at random (**note**: these are the weights of the linear model, they are *NOT* related in any way to the weights of the examples); each weight shall be sampled from U(-1,1);\n",
" - it evaluates the weighted loss $\\epsilon_t$ on the given dataset and flip the linear model if $\\epsilon_t > 0.5$;\n",
" - at prediction time it predicts +1 if $\\mathbf{x} \\cdot \\mathbf{w} > 0$; it predicts -1 otherwise."
]
},
{
"cell_type": "code",
"execution_count": 222,
"metadata": {},
"outputs": [],
"source": [
"class RandomLinearModel:\n",
" def loss(self, y, y_, sample_weight):\n",
" return sum([sample_weight[i] if y[i] != y_[i] else 0 for i in range(len(y))])\n",
" \n",
" def fit(self,X,y,sample_weight=[]):\n",
" self.w = np.random.rand(len(X[0])) * 2 - 1\n",
" if len(sample_weight) == 0:\n",
" sample_weight = [1 / len(X) for x in X]\n",
" if self.loss(y, self.predict(X), sample_weight) > 0.5:\n",
" self.w *= -1\n",
" return self\n",
" \n",
" def predict(self,X):\n",
" return np.sign(X.dot(self.w))"
]
},
{
"cell_type": "code",
"execution_count": 228,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.487\n"
]
}
],
"source": [
"rs = RandomLinearModel()\n",
"rs.fit(X_train, y_train)\n",
"predictions = rs.predict(X_test)\n",
"print(0.5 - y_test.dot(predictions)/(2 * len(y_test)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"2. Learn an AdaBoost model using the RandomLinearModel weak learner printing every $K$ iterations the weighted error and the current error of the ensemble (you are free to choose $K$ so to make your output just frequent enough to let you know what is happening but without flooding the console with messages). Evaluate the training and test error of the final ensemble model."
]
},
{
"cell_type": "code",
"execution_count": 229,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Weighted Error: 0.49524999999995933\n",
"Weighted Error: 0.4948541341954795\n",
"Weighted Error: 0.49729398392530305\n",
"Weighted Error: 0.49980867302257964\n",
"Weighted Error: 0.49683487146024025\n",
"Weighted Error: 0.49790489175815233\n",
"Weighted Error: 0.4940625587347454\n",
"Weighted Error: 0.4950371378338745\n",
"Weighted Error: 0.4909255291281916\n",
"Weighted Error: 0.4960331784908466\n"
]
}
],
"source": [
"rs = RandomLinearModel()\n",
"a = AdaBoost(rs,100)\n",
"a.fit(X_train,y_train)\n",
"\n",
"y_train_ = a.predict(X_train)\n",
"y_test_ = a.predict(X_test)"
]
},
{
"cell_type": "code",
"execution_count": 232,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Training Error: 0.462125\n",
"Test Error: 0.49375\n"
]
}
],
"source": [
"print(\"Training Error:\", 0.5 - y_train.dot(y_train_)/(2 * len(y_train)))\n",
"print(\"Test Error:\", 0.5 - y_test.dot(y_test_)/(2 * len(y_test)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"3. Write few paragraphs about what you think about the experiment and about the results you obtained."
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 1
}

385
anno3/apprendimento_automatico/esercizi/marco/AdaBoost.ipynb Normal file
View file

@ -0,0 +1,385 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import math\n",
"from copy import deepcopy\n",
"import sklearn.datasets\n",
"from sklearn.svm import SVC"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"X,y = sklearn.datasets.make_hastie_10_2()\n",
"X_train = X[0:8000,:]\n",
"y_train = y[0:8000]\n",
"X_test = X[8000:,:]\n",
"y_test = y[8000:]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"class SVC_:\n",
" def __init__(self, kernel=\"rbf\", degree=\"3\"):\n",
" self.svc = SVC(kernel=kernel, degree=degree)\n",
"\n",
" def fit(self, X, y, sample_weight=None):\n",
" if sample_weight is not None:\n",
" sample_weight = sample_weight * len(X)\n",
"\n",
" self.svc.fit(X,y,sample_weight=sample_weight)\n",
" return self\n",
"\n",
" def predict(self, X):\n",
" return self.svc.predict(X)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Exercise 1\n",
"\n",
"1. Implement the AdaBoost ensemble algorithm by completing the following code:"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"class AdaBoost:\n",
" def __init__(self, weakModel, T):\n",
" self.model = weakModel\n",
" self.models = []\n",
" self.T = T\n",
" self.a = []\n",
"\n",
" def fit(self, X, y):\n",
" w = np.array([1 / len(X) for x in X])\n",
" for t in range(self.T):\n",
" model = deepcopy(self.model)\n",
" model.fit(X, y, sample_weight = w)\n",
" predictions = model.predict(X)\n",
" self.models.append(model)\n",
" e = sum([w[i] if predictions[i] != y[i] else 0 for i in range(len(y))])\n",
" if t%10 == 0:\n",
" print(\"Weighted Error:\", e)\n",
" a = np.log((1 - e) / e) / 2\n",
" self.a.append(a)\n",
" w = np.array([w[i] * np.exp(-a * y[i] * predictions[i]) for i in range(len(w))])\n",
" w /= sum(w)\n",
" return self\n",
"\n",
" def predict(self, X):\n",
" return np.sign(sum([self.a[t] * self.models[t].predict(X) for t in range(self.T)]))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"In the implementation you are free to assume:\n",
"- that the problem is a binary classification problem with labels in $\\{-1, +1\\}$.\n",
"- that the weakModel can fit a weighted sample set by means of the call `weakModel.fit(X,y,sample_weight=w)` where `w` is a vector of length $|y|$."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"2. Test your implementation on the dataset loaded above and using an SVC with a polynomial kernel. "
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"8000"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(X_train)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Weighted Error: 0.3538749999999749\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n",
"C:\\Users\\galat\\.conda\\envs\\aaut\\lib\\site-packages\\sklearn\\svm\\base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
" \"avoid this warning.\", FutureWarning)\n"
]
}
],
"source": [
"weakModel = SVC_(kernel=\"poly\", degree=3)\n",
"adaboost = AdaBoost(weakModel, 10)\n",
"adaboost.fit(X_train, y_train)\n",
"y_test_ = adaboost.predict(X_test)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.22525\n"
]
}
],
"source": [
"print(0.5 - (y_test.dot(y_test_)) / (2 * len(y_test)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"3. evaluate the AdaBoost performances as usual by calculating the classification error and compare it with the classification error of the weak model.\n",
"\n",
"**Note 1**: \n",
"since the labels are bound to be in ${+1, -1}$, the classification error (i.e., the number of incorrectly classified examples over the total number of examples) can be easily computed as:\n",
"$$\n",
" error(y,y') = \\frac{N - y \\cdot y'}{2N} = \\frac{1}{2} - \\frac{y \\cdot y'}{2N},\n",
"$$\n",
"where $N$ is the total number of examples. The formula can be derived noticing that $y \\cdot y'$ calculates the number $N_c$ of examples correctly classified minus the number $N_{\\bar c}$ of examples incorrectly classified. We have then $y \\cdot y' = N_c - N_{\\bar c}$ and by noticing that $N = N_c + N_{\\bar c}$:\n",
"$$\n",
" N - y \\cdot y' = N_c + N_{\\bar c} - N_c + N_{\\bar c} = 2 N_{\\bar c} \\Rightarrow \\frac{N - y \\cdot y'}{2 N} = \\frac{N_{\\bar c}}{N}\n",
"$$\n",
"\n",
"**Note 2**:\n",
"do not forget to deepcopy your base model before fitting it to the new data\n",
"\n",
"**Note 3**:\n",
"The SVC model allows specifying weights, but it *does not* work well when weights are normalized (it works well when the weights are larger). The following class takes normalized weights and denormalize them before passing them to the SVC classifier:\n",
"\n",
"```python\n",
" class SVC_:\n",
" def __init__(self, kernel=\"rbf\", degree=\"3\"):\n",
" self.svc = SVC(kernel=kernel, degree=degree)\n",
"\n",
" def fit(self, X,y,sample_weight=None):\n",
" if sample_weight is not None:\n",
" sample_weight = sample_weight * len(X)\n",
"\n",
" self.svc.fit(X,y,sample_weight=sample_weight)\n",
" return self\n",
"\n",
" def predict(self, X):\n",
" return self.svc.predict(X)\n",
"```"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Exercise 2"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"1. Write a weak learner to be used with the AdaBoost algorithm you just wrote. The weak learner that you will implement is the most inaccurate weak learner possible: it basically works by extracting a linear model at random and trying to use that model to classify the examples. Being extracted at random the models it generates do not guarantee that the weighted error $\\epsilon_t$ is smaller than $0.5$. The algorithm solves this problem by flipping the decisions whenever it finds out that $\\epsilon_t > 0.5$ (i.e., if the weighted error is larger than $0.5$ it reverses the sign of all the weights so that the decision surface stays the same, but the regions where it predicts $+1$ and $-1$ are reversed).\n",
"\n",
" It shall work as follows:\n",
"\n",
" - it creates a random linear model by generating the needed weight vector $\\mathbf{w}$ at random (**note**: these are the weights of the linear model, they are *NOT* related in any way to the weights of the examples); each weight shall be sampled from U(-1,1);\n",
" - it evaluates the weighted loss $\\epsilon_t$ on the given dataset and flip the linear model if $\\epsilon_t > 0.5$;\n",
" - at prediction time it predicts +1 if $\\mathbf{x} \\cdot \\mathbf{w} > 0$; it predicts -1 otherwise."
]
},
{
"cell_type": "code",
"execution_count": 222,
"metadata": {},
"outputs": [],
"source": [
"class RandomLinearModel:\n",
" def loss(self, y, y_, sample_weight):\n",
" return sum([sample_weight[i] if y[i] != y_[i] else 0 for i in range(len(y))])\n",
" \n",
" def fit(self,X,y,sample_weight=[]):\n",
" self.w = np.random.rand(len(X[0])) * 2 - 1\n",
" if len(sample_weight) == 0:\n",
" sample_weight = [1 / len(X) for x in X]\n",
" if self.loss(y, self.predict(X), sample_weight) > 0.5:\n",
" self.w *= -1\n",
" return self\n",
" \n",
" def predict(self,X):\n",
" return np.sign(X.dot(self.w))"
]
},
{
"cell_type": "code",
"execution_count": 228,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0.487\n"
]
}
],
"source": [
"rs = RandomLinearModel()\n",
"rs.fit(X_train, y_train)\n",
"predictions = rs.predict(X_test)\n",
"print(0.5 - y_test.dot(predictions)/(2 * len(y_test)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"2. Learn an AdaBoost model using the RandomLinearModel weak learner printing every $K$ iterations the weighted error and the current error of the ensemble (you are free to choose $K$ so to make your output just frequent enough to let you know what is happening but without flooding the console with messages). Evaluate the training and test error of the final ensemble model."
]
},
{
"cell_type": "code",
"execution_count": 229,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Weighted Error: 0.49524999999995933\n",
"Weighted Error: 0.4948541341954795\n",
"Weighted Error: 0.49729398392530305\n",
"Weighted Error: 0.49980867302257964\n",
"Weighted Error: 0.49683487146024025\n",
"Weighted Error: 0.49790489175815233\n",
"Weighted Error: 0.4940625587347454\n",
"Weighted Error: 0.4950371378338745\n",
"Weighted Error: 0.4909255291281916\n",
"Weighted Error: 0.4960331784908466\n"
]
}
],
"source": [
"rs = RandomLinearModel()\n",
"a = AdaBoost(rs,100)\n",
"a.fit(X_train,y_train)\n",
"\n",
"y_train_ = a.predict(X_train)\n",
"y_test_ = a.predict(X_test)"
]
},
{
"cell_type": "code",
"execution_count": 232,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Training Error: 0.462125\n",
"Test Error: 0.49375\n"
]
}
],
"source": [
"print(\"Training Error:\", 0.5 - y_train.dot(y_train_)/(2 * len(y_train)))\n",
"print(\"Test Error:\", 0.5 - y_test.dot(y_test_)/(2 * len(y_test)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"3. Write few paragraphs about what you think about the experiment and about the results you obtained."
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 1
}

152
anno3/apprendimento_automatico/esercizi/marco/Datasets/3-clusters.csv Normal file
View file

@ -0,0 +1,152 @@
150,2
x,y
10,10
11,11
12,12
13,13
14,14
15,15
16,16
17,17
18,18
19,19
20,20
21,21
22,22
23,23
24,24
25,25
26,26
27,27
28,28
29,29
30,30
31,31
32,32
33,33
34,34
35,35
36,36
37,37
38,38
39,39
40,40
41,41
42,42
43,43
44,44
45,45
46,46
47,47
48,48
49,49
50,50
51,51
52,52
53,53
54,54
55,55
56,56
57,57
58,58
80,80
81,81
82,82
83,83
84,84
85,85
86,86
87,87
88,88
89,89
90,90
91,91
92,92
93,93
94,94
95,95
96,96
97,97
98,98
99,99
100,100
101,101
102,102
103,103
104,104
105,105
106,106
107,107
108,108
109,109
110,110
111,111
112,112
113,113
114,114
115,115
116,116
117,117
118,118
119,119
120,120
121,121
122,122
123,123
124,124
125,125
126,126
127,127
128,128
129,129
120,10
121,11
122,12
123,13
124,14
125,15
126,16
127,17
128,18
129,19
130,20
131,21
132,22
133,23
134,24
135,25
136,26
137,27
138,28
139,29
140,30
141,31
142,32
143,33
144,34
145,35
146,36
147,37
148,38
149,39
150,40
151,41
152,42
153,43
154,44
155,45
156,46
157,47
158,48
159,49
160,50
161,51
162,52
163,53
164,54
165,55
166,56
167,57
168,58
169,59
170,60
1 150 2
2 x y
3 10 10
4 11 11
5 12 12
6 13 13
7 14 14
8 15 15
9 16 16
10 17 17
11 18 18
12 19 19
13 20 20
14 21 21
15 22 22
16 23 23
17 24 24
18 25 25
19 26 26
20 27 27
21 28 28
22 29 29
23 30 30
24 31 31
25 32 32
26 33 33
27 34 34
28 35 35
29 36 36
30 37 37
31 38 38
32 39 39
33 40 40
34 41 41
35 42 42
36 43 43
37 44 44
38 45 45
39 46 46
40 47 47
41 48 48
42 49 49
43 50 50
44 51 51
45 52 52
46 53 53
47 54 54
48 55 55
49 56 56
50 57 57
51 58 58
52 80 80
53 81 81
54 82 82
55 83 83
56 84 84
57 85 85
58 86 86
59 87 87
60 88 88
61 89 89
62 90 90
63 91 91
64 92 92
65 93 93
66 94 94
67 95 95
68 96 96
69 97 97
70 98 98
71 99 99
72 100 100
73 101 101
74 102 102
75 103 103
76 104 104
77 105 105
78 106 106
79 107 107
80 108 108
81 109 109
82 110 110
83 111 111
84 112 112
85 113 113
86 114 114
87 115 115
88 116 116
89 117 117
90 118 118
91 119 119
92 120 120
93 121 121
94 122 122
95 123 123
96 124 124
97 125 125
98 126 126
99 127 127
100 128 128
101 129 129
102 120 10
103 121 11
104 122 12
105 123 13
106 124 14
107 125 15
108 126 16
109 127 17
110 128 18
111 129 19
112 130 20
113 131 21
114 132 22
115 133 23
116 134 24
117 135 25
118 136 26
119 137 27
120 138 28
121 139 29
122 140 30
123 141 31
124 142 32
125 143 33
126 144 34
127 145 35
128 146 36
129 147 37
130 148 38
131 149 39
132 150 40
133 151 41
134 152 42
135 153 43
136 154 44
137 155 45
138 156 46
139 157 47
140 158 48
141 159 49
142 160 50
143 161 51
144 162 52
145 163 53
146 164 54
147 165 55
148 166 56
149 167 57
150 168 58
151 169 59
152 170 60

86560
anno3/apprendimento_automatico/esercizi/marco/Datasets/CURE-complete.csv Normal file
View file

File diff suppressed because it is too large Load diff

6120
anno3/apprendimento_automatico/esercizi/marco/Datasets/dataset-DBSCAN.csv Normal file
View file

File diff suppressed because it is too large Load diff

BIN
anno3/apprendimento_automatico/esercizi/marco/aaut-merged.pdf Normal file
View file

Binary file not shown.

1810
anno3/apprendimento_automatico/esercizi/marco/classification_iris_aa_19_20-checkpoint.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

1810
anno3/apprendimento_automatico/esercizi/marco/classification_iris_aa_19_20.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

1807
anno3/apprendimento_automatico/esercizi/marco/classification_iris_knn_aa_19_20-checkpoint.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

1807
anno3/apprendimento_automatico/esercizi/marco/classification_iris_knn_aa_19_20.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

186957
anno3/apprendimento_automatico/esercizi/marco/clustering-copy_for_students_aa_19_20-checkpoint.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

187191
anno3/apprendimento_automatico/esercizi/marco/clustering-copy_for_students_aa_19_20.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

491
anno3/apprendimento_automatico/esercizi/marco/coverage_plots-checkpoint.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

491
anno3/apprendimento_automatico/esercizi/marco/coverage_plots.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

840
anno3/apprendimento_automatico/esercizi/marco/least_squares-checkpoint.ipynb Normal file
View file

@ -0,0 +1,840 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Experimenting with least squares and its variants"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"from sklearn import datasets\n",
"from scipy.optimize import fmin_bfgs\n",
"import numpy as np\n",
"from numpy.linalg import norm\n",
"from numpy.linalg import inv"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Data preparation"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"boston = datasets.load_boston()\n",
"data = np.array(boston.data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The boston dataset is one of the standard regression problems used to experiment with learning algorithms. Below you can find the dataset description"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
".. _boston_dataset:\n",
"\n",
"Boston house prices dataset\n",
"---------------------------\n",
"\n",
"**Data Set Characteristics:** \n",
"\n",
" :Number of Instances: 506 \n",
"\n",
" :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.\n",
"\n",
" :Attribute Information (in order):\n",
" - CRIM per capita crime rate by town\n",
" - ZN proportion of residential land zoned for lots over 25,000 sq.ft.\n",
" - INDUS proportion of non-retail business acres per town\n",
" - CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)\n",
" - NOX nitric oxides concentration (parts per 10 million)\n",
" - RM average number of rooms per dwelling\n",
" - AGE proportion of owner-occupied units built prior to 1940\n",
" - DIS weighted distances to five Boston employment centres\n",
" - RAD index of accessibility to radial highways\n",
" - TAX full-value property-tax rate per $10,000\n",
" - PTRATIO pupil-teacher ratio by town\n",
" - B 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town\n",
" - LSTAT % lower status of the population\n",
" - MEDV Median value of owner-occupied homes in $1000's\n",
"\n",
" :Missing Attribute Values: None\n",
"\n",
" :Creator: Harrison, D. and Rubinfeld, D.L.\n",
"\n",
"This is a copy of UCI ML housing dataset.\n",
"https://archive.ics.uci.edu/ml/machine-learning-databases/housing/\n",
"\n",
"\n",
"This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.\n",
"\n",
"The Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic\n",
"prices and the demand for clean air', J. Environ. Economics & Management,\n",
"vol.5, 81-102, 1978. Used in Belsley, Kuh & Welsch, 'Regression diagnostics\n",
"...', Wiley, 1980. N.B. Various transformations are used in the table on\n",
"pages 244-261 of the latter.\n",
"\n",
"The Boston house-price data has been used in many machine learning papers that address regression\n",
"problems. \n",
" \n",
".. topic:: References\n",
"\n",
" - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261.\n",
" - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.\n",
"\n"
]
}
],
"source": [
"print(boston.DESCR)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First step to apply the formulae we learnt during the lectures is to rewrite the dataset in homogeneous coordinates (i.e., we append a column of 1 to the matrix containing the examples):"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.]])"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"t = np.ones(len(data)).reshape(len(data),1)\n",
"data = np.append(data, t, 1)\n",
"target = np.array(boston.target)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We now divide the data into a training set $X$ and a test set $X_\\textrm{test}$."
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"X,y = data[0:400,:], target[0:400]\n",
"X_test, y_test = data[400:,:], target[400:]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Exercise"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"1. Calculate the least square solution (to the regression problem outlined above) and evaluate its performances on the training set and on the test set.\n",
"1. Calculate the ridge regression solution (set lambda to 0.01) and evaluate its performances on the training set and on test set.\n",
"1. Calculate the lasso regression solution and evaluate its performances on the training set and on the test set."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Notes"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"- Here it follows a list of functions you may want to use (the required packages are already imported at the beginning of this notebook) along with a very brief explanation of their purpose (`help(nomefun)` will provide you more information about function `nomefun`):\n",
" - `transpose`: matrix transposition (e.g., `transpose(X)`)\n",
" - `dot`: matrix multiplication (e.g., `X.dot(X2)`) \n",
" - `inv`: matrix inversion (e.g., `inv(X)`)\n",
"- to solve the lasso problem you will need to perform a numerical minimization of the associated loss function (as you know, a closed form solution does not exist). There are many numerical optimization algorithms available in the scipy package. My suggestion is to use `fmin_bfgs`. Here it follows an example of how to use it:\n",
" ```python\n",
" def f(w):\n",
" return w[0]**2 + w[1]**2 + w[0] + w[1]\n",
" \n",
" w = fmin_bfgs(f, [0,0])\n",
" ```\n",
" note that the function may (and should) reference your data variables (i.e., $X$ and $y$).\n",
"- to evaluate the performances of your solutions use the $S$ statistic:\n",
" $$\n",
" S = \\sqrt{ \\frac{1}{n} \\sum_{i=1}^n (y_i' - y_i)^2 }\n",
" $$\n",
" where $y'_i$ is your model prediction for the i-th example, and $n$ is the number of examples."
]
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {},
"outputs": [],
"source": [
"import math"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {},
"outputs": [],
"source": [
"def least_squares(X, y):\n",
" return inv(np.transpose(X).dot(X)).dot(np.transpose(X)).dot(y)"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [],
"source": [
"def ridge_regression(X, y, lam):\n",
" return inv(np.transpose(X).dot(X) + np.identity(len(X[0])).dot(lam)).dot(np.transpose(X)).dot(y)"
]
},
{
"cell_type": "code",
"execution_count": 83,
"metadata": {},
"outputs": [],
"source": [
"def lasso(w, X, y, lam):\n",
" return np.transpose(y - X.dot(w)).dot(y - X.dot(w)) + lam * sum(w)"
]
},
{
"cell_type": "code",
"execution_count": 79,
"metadata": {},
"outputs": [],
"source": [
"def lasso_regression(X, y, lam):\n",
" return fmin_bfgs(lasso, np.zeros(len(X[0])), args = (X, y, lam))"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [],
"source": [
"def s_statistics(actual, predicted):\n",
" return math.sqrt(sum([(predicted[i] - actual[i])**2 for i in range(len(actual))]) / len(actual))"
]
},
{
"cell_type": "code",
"execution_count": 80,
"metadata": {},
"outputs": [],
"source": [
"def predict(X, w):\n",
" return X.dot(w)"
]
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: Desired error not necessarily achieved due to precision loss.\n",
" Current function value: 8922.270593\n",
" Iterations: 19\n",
" Function evaluations: 577\n",
" Gradient evaluations: 36\n",
"Least squares training set s statistics: 4.7228408383263805\n",
"Least squares test set s statistics: 6.155792280414106\n",
"Ridge regression training set s statistics: 4.7228952650983205\n",
"Ridge regression test set s statistics: 6.141787930906379\n",
"Lasso regression training set s statistics: 4.722840843957779\n",
"Lasso regression test set s statistics: 6.1558291277697\n"
]
}
],
"source": [
"w_least = least_squares(X, y)\n",
"w_ridge = ridge_regression(X, y, 0.01)\n",
"w_lasso = lasso_regression(X, y, 0.01)\n",
"print(\"Least squares training set s statistics:\", s_statistics(y, predict(X, w_least)))\n",
"print(\"Least squares test set s statistics:\", s_statistics(y_test, predict(X_test, w_least)))\n",
"print(\"Ridge regression training set s statistics:\", s_statistics(y, predict(X, w_ridge)))\n",
"print(\"Ridge regression test set s statistics:\", s_statistics(y_test, predict(X_test, w_ridge)))\n",
"print(\"Lasso regression training set s statistics:\", s_statistics(y, predict(X, w_lasso)))\n",
"print(\"Lasso regression test set s statistics:\", s_statistics(y_test, predict(X_test, w_lasso)))"
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 1
}

840
anno3/apprendimento_automatico/esercizi/marco/least_squares.ipynb Normal file
View file

@ -0,0 +1,840 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Experimenting with least squares and its variants"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"from sklearn import datasets\n",
"from scipy.optimize import fmin_bfgs\n",
"import numpy as np\n",
"from numpy.linalg import norm\n",
"from numpy.linalg import inv"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Data preparation"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"boston = datasets.load_boston()\n",
"data = np.array(boston.data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The boston dataset is one of the standard regression problems used to experiment with learning algorithms. Below you can find the dataset description"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
".. _boston_dataset:\n",
"\n",
"Boston house prices dataset\n",
"---------------------------\n",
"\n",
"**Data Set Characteristics:** \n",
"\n",
" :Number of Instances: 506 \n",
"\n",
" :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.\n",
"\n",
" :Attribute Information (in order):\n",
" - CRIM per capita crime rate by town\n",
" - ZN proportion of residential land zoned for lots over 25,000 sq.ft.\n",
" - INDUS proportion of non-retail business acres per town\n",
" - CHAS Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)\n",
" - NOX nitric oxides concentration (parts per 10 million)\n",
" - RM average number of rooms per dwelling\n",
" - AGE proportion of owner-occupied units built prior to 1940\n",
" - DIS weighted distances to five Boston employment centres\n",
" - RAD index of accessibility to radial highways\n",
" - TAX full-value property-tax rate per $10,000\n",
" - PTRATIO pupil-teacher ratio by town\n",
" - B 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town\n",
" - LSTAT % lower status of the population\n",
" - MEDV Median value of owner-occupied homes in $1000's\n",
"\n",
" :Missing Attribute Values: None\n",
"\n",
" :Creator: Harrison, D. and Rubinfeld, D.L.\n",
"\n",
"This is a copy of UCI ML housing dataset.\n",
"https://archive.ics.uci.edu/ml/machine-learning-databases/housing/\n",
"\n",
"\n",
"This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.\n",
"\n",
"The Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic\n",
"prices and the demand for clean air', J. Environ. Economics & Management,\n",
"vol.5, 81-102, 1978. Used in Belsley, Kuh & Welsch, 'Regression diagnostics\n",
"...', Wiley, 1980. N.B. Various transformations are used in the table on\n",
"pages 244-261 of the latter.\n",
"\n",
"The Boston house-price data has been used in many machine learning papers that address regression\n",
"problems. \n",
" \n",
".. topic:: References\n",
"\n",
" - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261.\n",
" - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.\n",
"\n"
]
}
],
"source": [
"print(boston.DESCR)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"First step to apply the formulae we learnt during the lectures is to rewrite the dataset in homogeneous coordinates (i.e., we append a column of 1 to the matrix containing the examples):"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.],\n",
" [1.]])"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"t = np.ones(len(data)).reshape(len(data),1)\n",
"data = np.append(data, t, 1)\n",
"target = np.array(boston.target)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We now divide the data into a training set $X$ and a test set $X_\\textrm{test}$."
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"X,y = data[0:400,:], target[0:400]\n",
"X_test, y_test = data[400:,:], target[400:]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Exercise"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"1. Calculate the least square solution (to the regression problem outlined above) and evaluate its performances on the training set and on the test set.\n",
"1. Calculate the ridge regression solution (set lambda to 0.01) and evaluate its performances on the training set and on test set.\n",
"1. Calculate the lasso regression solution and evaluate its performances on the training set and on the test set."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Notes"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"- Here it follows a list of functions you may want to use (the required packages are already imported at the beginning of this notebook) along with a very brief explanation of their purpose (`help(nomefun)` will provide you more information about function `nomefun`):\n",
" - `transpose`: matrix transposition (e.g., `transpose(X)`)\n",
" - `dot`: matrix multiplication (e.g., `X.dot(X2)`) \n",
" - `inv`: matrix inversion (e.g., `inv(X)`)\n",
"- to solve the lasso problem you will need to perform a numerical minimization of the associated loss function (as you know, a closed form solution does not exist). There are many numerical optimization algorithms available in the scipy package. My suggestion is to use `fmin_bfgs`. Here it follows an example of how to use it:\n",
" ```python\n",
" def f(w):\n",
" return w[0]**2 + w[1]**2 + w[0] + w[1]\n",
" \n",
" w = fmin_bfgs(f, [0,0])\n",
" ```\n",
" note that the function may (and should) reference your data variables (i.e., $X$ and $y$).\n",
"- to evaluate the performances of your solutions use the $S$ statistic:\n",
" $$\n",
" S = \\sqrt{ \\frac{1}{n} \\sum_{i=1}^n (y_i' - y_i)^2 }\n",
" $$\n",
" where $y'_i$ is your model prediction for the i-th example, and $n$ is the number of examples."
]
},
{
"cell_type": "code",
"execution_count": 65,
"metadata": {},
"outputs": [],
"source": [
"import math"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {},
"outputs": [],
"source": [
"def least_squares(X, y):\n",
" return inv(np.transpose(X).dot(X)).dot(np.transpose(X)).dot(y)"
]
},
{
"cell_type": "code",
"execution_count": 67,
"metadata": {},
"outputs": [],
"source": [
"def ridge_regression(X, y, lam):\n",
" return inv(np.transpose(X).dot(X) + np.identity(len(X[0])).dot(lam)).dot(np.transpose(X)).dot(y)"
]
},
{
"cell_type": "code",
"execution_count": 83,
"metadata": {},
"outputs": [],
"source": [
"def lasso(w, X, y, lam):\n",
" return np.transpose(y - X.dot(w)).dot(y - X.dot(w)) + lam * sum(w)"
]
},
{
"cell_type": "code",
"execution_count": 79,
"metadata": {},
"outputs": [],
"source": [
"def lasso_regression(X, y, lam):\n",
" return fmin_bfgs(lasso, np.zeros(len(X[0])), args = (X, y, lam))"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {},
"outputs": [],
"source": [
"def s_statistics(actual, predicted):\n",
" return math.sqrt(sum([(predicted[i] - actual[i])**2 for i in range(len(actual))]) / len(actual))"
]
},
{
"cell_type": "code",
"execution_count": 80,
"metadata": {},
"outputs": [],
"source": [
"def predict(X, w):\n",
" return X.dot(w)"
]
},
{
"cell_type": "code",
"execution_count": 89,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: Desired error not necessarily achieved due to precision loss.\n",
" Current function value: 8922.270593\n",
" Iterations: 19\n",
" Function evaluations: 577\n",
" Gradient evaluations: 36\n",
"Least squares training set s statistics: 4.7228408383263805\n",
"Least squares test set s statistics: 6.155792280414106\n",
"Ridge regression training set s statistics: 4.7228952650983205\n",
"Ridge regression test set s statistics: 6.141787930906379\n",
"Lasso regression training set s statistics: 4.722840843957779\n",
"Lasso regression test set s statistics: 6.1558291277697\n"
]
}
],
"source": [
"w_least = least_squares(X, y)\n",
"w_ridge = ridge_regression(X, y, 0.01)\n",
"w_lasso = lasso_regression(X, y, 0.01)\n",
"print(\"Least squares training set s statistics:\", s_statistics(y, predict(X, w_least)))\n",
"print(\"Least squares test set s statistics:\", s_statistics(y_test, predict(X_test, w_least)))\n",
"print(\"Ridge regression training set s statistics:\", s_statistics(y, predict(X, w_ridge)))\n",
"print(\"Ridge regression test set s statistics:\", s_statistics(y_test, predict(X_test, w_ridge)))\n",
"print(\"Lasso regression training set s statistics:\", s_statistics(y, predict(X, w_lasso)))\n",
"print(\"Lasso regression test set s statistics:\", s_statistics(y_test, predict(X_test, w_lasso)))"
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 1
}

24
anno3/apprendimento_automatico/esercizi/marco/my_iris_predictions Normal file
View file

@ -0,0 +1,24 @@
digraph Tree {
node [shape=box] ;
0 [label="X[2] <= 2.45\nentropy = 1.585\nsamples = 150\nvalue = [50, 50, 50]"] ;
1 [label="entropy = 0.0\nsamples = 50\nvalue = [50, 0, 0]"] ;
0 -> 1 [labeldistance=2.5, labelangle=45, headlabel="True"] ;
2 [label="X[3] <= 1.75\nentropy = 1.0\nsamples = 100\nvalue = [0, 50, 50]"] ;
0 -> 2 [labeldistance=2.5, labelangle=-45, headlabel="False"] ;
3 [label="X[2] <= 4.95\nentropy = 0.445\nsamples = 54\nvalue = [0, 49, 5]"] ;
2 -> 3 ;
4 [label="X[0] <= 5.15\nentropy = 0.146\nsamples = 48\nvalue = [0, 47, 1]"] ;
3 -> 4 ;
5 [label="entropy = 0.722\nsamples = 5\nvalue = [0, 4, 1]"] ;
4 -> 5 ;
6 [label="entropy = 0.0\nsamples = 43\nvalue = [0, 43, 0]"] ;
4 -> 6 ;
7 [label="entropy = 0.918\nsamples = 6\nvalue = [0, 2, 4]"] ;
3 -> 7 ;
8 [label="X[2] <= 4.95\nentropy = 0.151\nsamples = 46\nvalue = [0, 1, 45]"] ;
2 -> 8 ;
9 [label="entropy = 0.65\nsamples = 6\nvalue = [0, 1, 5]"] ;
8 -> 9 ;
10 [label="entropy = 0.0\nsamples = 40\nvalue = [0, 0, 40]"] ;
8 -> 10 ;
}

BIN
anno3/apprendimento_automatico/esercizi/marco/my_iris_predictions.pdf Normal file
View file

Binary file not shown.

324
anno3/apprendimento_automatico/esercizi/marco/one_vs_rest-checkpoint.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

324
anno3/apprendimento_automatico/esercizi/marco/one_vs_rest.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

617
anno3/apprendimento_automatico/esercizi/marco/svm-checkpoint.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

617
anno3/apprendimento_automatico/esercizi/marco/svm.ipynb Normal file
View file

File diff suppressed because one or more lines are too long

3
media.py
View file

@ -9,7 +9,8 @@ voti = [
(24, 9), # ling. formali
(30, 6), # mcad
(30, 6), # scpd
(27, 9), # vpc
(27, 9), # vpc
(30, 6), # progmobile
]
crediti, voto = sum(map(lambda x: x[1], voti)), sum(map(lambda x: x[0]*x[1], voti))