diff --git a/Problem 1.ipynb b/Problem 1.ipynb new file mode 100644 index 0000000..df82679 --- /dev/null +++ b/Problem 1.ipynb @@ -0,0 +1,227 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2) Plot the data " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "data=pd.read_table('chickwts.txt',sep=',')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.boxplot(by='feed',column='weight',grid=False,figsize=(15,15))\n", + "# I am not sure why somehow there is no figure in Jupyter but I did get a barplot in python. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "## 3) A hypothesis test\n", + "\n", + "### Null hypothesis: the weight of chick fed by soybean has no significant difference with the weight of chick fed by sunflower\n", + "### Alternative hypothesis: the weight of chick fed by soybean has significant difference with the weight of chick fed by sunflower" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4) Hypothesis test" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "le=LabelEncoder()\n", + "le.fit(data['feed'])\n", + "data['Index']=le.transform(data['feed'])\n", + "x=data[(data['feed']=='soybean')|(data['feed']=='sunflower')]['Index']\n", + "y=data[(data['feed']=='soybean')|(data['feed']=='sunflower')]['weight']" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimization terminated successfully.\n", + " Current function value: 145.240592\n", + " Iterations: 85\n", + " Function evaluations: 162\n" + ] + }, + { + "data": { + "text/plain": [ + "array([ 284.49999051, 64.53691755])" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "from scipy.optimize import minimize\n", + "from scipy.stats import norm\n", + "\n", + "# null hypothesis\n", + "def null(p,obs):\n", + " B0=p[0]\n", + " sigma=p[1]\n", + " \n", + " expected=B0\n", + " nll=-1*norm(expected,sigma).logpdf(obs).sum()\n", + " return nll\n", + "\n", + "initialGuess1=np.array([1,1])\n", + "fitNull=minimize(null,initialGuess1,method=\"Nelder-Mead\",options={'disp':True},args=(y))\n", + "fitNull.x" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimization terminated successfully.\n", + " Current function value: 138.469162\n", + " Iterations: 194\n", + " Function evaluations: 360\n" + ] + }, + { + "data": { + "text/plain": [ + "array([-83.52384926, 82.48810462, 49.73945446])" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# alternative hypothesis\n", + "def alter(p,obs1,obs2):\n", + " B0=p[0]\n", + " B1=p[1]\n", + " sigma=p[2]\n", + " \n", + " expected=B0+B1*obs1\n", + " nll=-1*norm(expected,sigma).logpdf(obs2).sum()\n", + " return nll\n", + "\n", + "initialGuess1=np.array([1,1,1])\n", + "fitalter=minimize(alter,initialGuess1,method=\"Nelder-Mead\",options={'disp':True},args=(x,y))\n", + "fitalter.x" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0002331767286918307" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from scipy.stats import chi2\n", + "D=2*(fitNull.fun-fitalter.fun)\n", + "1-chi2.cdf(x=D,df=1) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5) Hypothesis test result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## If we consider alfa=0.05, then p value is smaller than 0.05, we can reject the null hyphothesis and conclude that the weight of chick fed by soybean has significant difference with the weight of chick fed by sunflower. " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Problem 2.ipynb b/Problem 2.ipynb new file mode 100644 index 0000000..9ee0cbb --- /dev/null +++ b/Problem 2.ipynb @@ -0,0 +1,126 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1) Times after noon but before midnight, reported in 24 hours or \"military\" format \n", + "\n", + "### Times after noon and before midnight, so the numbes before comma should be larger than 12 and smaller than 24. Meanwhile, the number for minute should be in the range of 0-60." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['15:30']" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import re as re\n", + "reg1=re.compile('([1][2-9]\\:([0-5][0-9]|[6][0]))|([2][0-3]\\:([0-5][0-9]|[6][0]))')\n", + "testset1=['15:30','24:00']\n", + "filter(reg1.match,testset1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2) Genus species names \n", + "\n", + "### First letter should be capitalized, then it comes with the \".\", then six letters in the lower case. " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['H.sapien']" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "reg2=re.compile('[A-Z]{1}\\.[a-z]{6}')\n", + "testset2=['H.sapien','h.sapien']\n", + "filter(reg2.match,testset2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3) Social security numbers\n", + "\n", + "### Three numbers, then \"-\", then two numbers, then \"-\", then four numbers " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['389-05-4771']" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "reg3=re.compile('[0-9]{3}\\-[0-9]{2}\\-[0-9]{4}')\n", + "testset3=['389-05-4771','38-051-41']\n", + "filter(reg3.match,testset3)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Problem+1.html b/Problem+1.html new file mode 100644 index 0000000..c7063b3 --- /dev/null +++ b/Problem+1.html @@ -0,0 +1,12046 @@ + + + +Problem 1 + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+
+
+
+

Challenge 1

+
+
+
+
+
+
+
+
+

2) Plot the data

+
+
+
+
+
+
In [2]:
+
+
+
import pandas as pd
+data=pd.read_table('chickwts.txt',sep=',')
+
+ +
+
+
+ +
+
+
+
In [6]:
+
+
+
data.boxplot(by='feed',column='weight',grid=False,figsize=(15,15))
+# I am not sure why somehow there is no figure in Jupyter but I did get a barplot in python. 
+
+ +
+
+
+ +
+
+ + +
+
Out[6]:
+ + + +
+
<matplotlib.axes._subplots.AxesSubplot at 0x115b369d0>
+
+ +
+ +
+
+ +
+
+
+
+
+
+

3) A hypothesis test

Null hypothesis: the weight of chick fed by soybean has no significant difference with the weight of chick fed by sunflower

Alternative hypothesis: the weight of chick fed by soybean has significant difference with the weight of chick fed by sunflower

+
+
+
+
+
+
+
+
+

4) Hypothesis test

+
+
+
+
+
+
In [31]:
+
+
+
le=LabelEncoder()
+le.fit(data['feed'])
+data['Index']=le.transform(data['feed'])
+x=data[(data['feed']=='soybean')|(data['feed']=='sunflower')]['Index']
+y=data[(data['feed']=='soybean')|(data['feed']=='sunflower')]['weight']
+
+ +
+
+
+ +
+
+
+
In [28]:
+
+
+
import numpy as np
+from scipy.optimize import minimize
+from scipy.stats import norm
+
+# null hypothesis
+def null(p,obs):
+    B0=p[0]
+    sigma=p[1]
+    
+    expected=B0
+    nll=-1*norm(expected,sigma).logpdf(obs).sum()
+    return nll
+
+initialGuess1=np.array([1,1])
+fitNull=minimize(null,initialGuess1,method="Nelder-Mead",options={'disp':True},args=(y))
+fitNull.x
+
+ +
+
+
+ +
+
+ + +
+
+ +
+
Optimization terminated successfully.
+         Current function value: 145.240592
+         Iterations: 85
+         Function evaluations: 162
+
+
+
+ +
+
Out[28]:
+ + + +
+
array([ 284.49999051,   64.53691755])
+
+ +
+ +
+
+ +
+
+
+
In [29]:
+
+
+
# alternative hypothesis
+def alter(p,obs1,obs2):
+    B0=p[0]
+    B1=p[1]
+    sigma=p[2]
+    
+    expected=B0+B1*obs1
+    nll=-1*norm(expected,sigma).logpdf(obs2).sum()
+    return nll
+
+initialGuess1=np.array([1,1,1])
+fitalter=minimize(alter,initialGuess1,method="Nelder-Mead",options={'disp':True},args=(x,y))
+fitalter.x
+
+ +
+
+
+ +
+
+ + +
+
+ +
+
Optimization terminated successfully.
+         Current function value: 138.469162
+         Iterations: 194
+         Function evaluations: 360
+
+
+
+ +
+
Out[29]:
+ + + +
+
array([-83.52384926,  82.48810462,  49.73945446])
+
+ +
+ +
+
+ +
+
+
+
In [30]:
+
+
+
from scipy.stats import chi2
+D=2*(fitNull.fun-fitalter.fun)
+1-chi2.cdf(x=D,df=1) 
+
+ +
+
+
+ +
+
+ + +
+
Out[30]:
+ + + +
+
0.0002331767286918307
+
+ +
+ +
+
+ +
+
+
+
+
+
+

5) Hypothesis test result

+
+
+
+
+
+
+
+
+

If we consider alfa=0.05, then p value is smaller than 0.05, we can reject the null hyphothesis and conclude that the weight of chick fed by soybean has significant difference with the weight of chick fed by sunflower.

+
+
+
+
+
+ + + + + + diff --git a/Problem+2.html b/Problem+2.html new file mode 100644 index 0000000..4f49248 --- /dev/null +++ b/Problem+2.html @@ -0,0 +1,11919 @@ + + + +Problem 2 + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+
+
+
+

Challenge 2

+
+
+
+
+
+
+
+
+

1) Times after noon but before midnight, reported in 24 hours or "military" format

Times after noon and before midnight, so the numbes before comma should be larger than 12 and smaller than 24. Meanwhile, the number for minute should be in the range of 0-60.

+
+
+
+
+
+
In [12]:
+
+
+
import re as re
+reg1=re.compile('([1][2-9]\:([0-5][0-9]|[6][0]))|([2][0-3]\:([0-5][0-9]|[6][0]))')
+testset1=['15:30','24:00']
+filter(reg1.match,testset1)
+
+ +
+
+
+ +
+
+ + +
+
Out[12]:
+ + + +
+
['15:30']
+
+ +
+ +
+
+ +
+
+
+
+
+
+

2) Genus species names

First letter should be capitalized, then it comes with the ".", then six letters in the lower case.

+
+
+
+
+
+
In [13]:
+
+
+
reg2=re.compile('[A-Z]{1}\.[a-z]{6}')
+testset2=['H.sapien','h.sapien']
+filter(reg2.match,testset2)
+
+ +
+
+
+ +
+
+ + +
+
Out[13]:
+ + + +
+
['H.sapien']
+
+ +
+ +
+
+ +
+
+
+
+
+
+

3) Social security numbers

Three numbers, then "-", then two numbers, then "-", then four numbers

+
+
+
+
+
+
In [14]:
+
+
+
reg3=re.compile('[0-9]{3}\-[0-9]{2}\-[0-9]{4}')
+testset3=['389-05-4771','38-051-41']
+filter(reg3.match,testset3)
+
+ +
+
+
+ +
+
+ + +
+
Out[14]:
+ + + +
+
['389-05-4771']
+
+ +
+ +
+
+ +
+
+
+ + + + + + diff --git a/Week 12 Question 1.ipynb b/Week 12 Question 1.ipynb new file mode 100644 index 0000000..69186c1 --- /dev/null +++ b/Week 12 Question 1.ipynb @@ -0,0 +1,139 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#Question 1\n", + "import pandas as pd\n", + "import numpy\n", + "from scipy.optimize import minimize\n", + "from scipy.stats import norm\n", + "import scipy.integrate as spi\n", + "from scipy.integrate import odeint\n", + "import matplotlib.pyplot as plt\n", + "import re\n", + "import os\n", + "from plotnine import *" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "InFile=open(\"chickwts.txt\",\"r\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "count 2.000000\n", + "mean 36.500000\n", + "std 50.204581\n", + "min 1.000000\n", + "25% 18.750000\n", + "50% 36.500000\n", + "75% 54.250000\n", + "max 72.000000\n", + "dtype: float64" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "s = pd.Series([1,72])\n", + "s.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "count 4\n", + "unique 3\n", + "top a\n", + "freq 2\n", + "dtype: object" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "s = pd.Series(['a', 'a', 'b', 'c'])\n", + "s.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'df' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdescribe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minclude\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'all'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined" + ] + } + ], + "source": [ + "df.describe(include='all')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}