From 3995f9d550d0fe63984ea3c48284b1336076b25d Mon Sep 17 00:00:00 2001 From: mbuynak <31928750+mbuynak@users.noreply.github.com> Date: Fri, 17 Nov 2017 11:23:12 -0500 Subject: [PATCH 1/3] Add files via upload --- Week 12 Question 1.ipynb | 139 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 Week 12 Question 1.ipynb diff --git a/Week 12 Question 1.ipynb b/Week 12 Question 1.ipynb new file mode 100644 index 0000000..69186c1 --- /dev/null +++ b/Week 12 Question 1.ipynb @@ -0,0 +1,139 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#Question 1\n", + "import pandas as pd\n", + "import numpy\n", + "from scipy.optimize import minimize\n", + "from scipy.stats import norm\n", + "import scipy.integrate as spi\n", + "from scipy.integrate import odeint\n", + "import matplotlib.pyplot as plt\n", + "import re\n", + "import os\n", + "from plotnine import *" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "InFile=open(\"chickwts.txt\",\"r\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "count 2.000000\n", + "mean 36.500000\n", + "std 50.204581\n", + "min 1.000000\n", + "25% 18.750000\n", + "50% 36.500000\n", + "75% 54.250000\n", + "max 72.000000\n", + "dtype: float64" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "s = pd.Series([1,72])\n", + "s.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "count 4\n", + "unique 3\n", + "top a\n", + "freq 2\n", + "dtype: object" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "s = pd.Series(['a', 'a', 'b', 'c'])\n", + "s.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'df' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdescribe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minclude\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'all'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'df' is not defined" + ] + } + ], + "source": [ + "df.describe(include='all')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 4387570ffc27ee80afd8f6dfb994ab69ca0dc113 Mon Sep 17 00:00:00 2001 From: yunluyingying Date: Fri, 17 Nov 2017 11:26:25 -0500 Subject: [PATCH 2/3] Problem 2 --- Problem 2.ipynb | 126 + Problem+2.html | 11919 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 12045 insertions(+) create mode 100644 Problem 2.ipynb create mode 100644 Problem+2.html diff --git a/Problem 2.ipynb b/Problem 2.ipynb new file mode 100644 index 0000000..9ee0cbb --- /dev/null +++ b/Problem 2.ipynb @@ -0,0 +1,126 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1) Times after noon but before midnight, reported in 24 hours or \"military\" format \n", + "\n", + "### Times after noon and before midnight, so the numbes before comma should be larger than 12 and smaller than 24. Meanwhile, the number for minute should be in the range of 0-60." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['15:30']" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import re as re\n", + "reg1=re.compile('([1][2-9]\\:([0-5][0-9]|[6][0]))|([2][0-3]\\:([0-5][0-9]|[6][0]))')\n", + "testset1=['15:30','24:00']\n", + "filter(reg1.match,testset1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2) Genus species names \n", + "\n", + "### First letter should be capitalized, then it comes with the \".\", then six letters in the lower case. " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['H.sapien']" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "reg2=re.compile('[A-Z]{1}\\.[a-z]{6}')\n", + "testset2=['H.sapien','h.sapien']\n", + "filter(reg2.match,testset2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3) Social security numbers\n", + "\n", + "### Three numbers, then \"-\", then two numbers, then \"-\", then four numbers " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['389-05-4771']" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "reg3=re.compile('[0-9]{3}\\-[0-9]{2}\\-[0-9]{4}')\n", + "testset3=['389-05-4771','38-051-41']\n", + "filter(reg3.match,testset3)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Problem+2.html b/Problem+2.html new file mode 100644 index 0000000..4f49248 --- /dev/null +++ b/Problem+2.html @@ -0,0 +1,11919 @@ + + + +Problem 2 + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+
+
+
+

Challenge 2

+
+
+
+
+
+
+
+
+

1) Times after noon but before midnight, reported in 24 hours or "military" format

Times after noon and before midnight, so the numbes before comma should be larger than 12 and smaller than 24. Meanwhile, the number for minute should be in the range of 0-60.

+
+
+
+
+
+
In [12]:
+
+
+
import re as re
+reg1=re.compile('([1][2-9]\:([0-5][0-9]|[6][0]))|([2][0-3]\:([0-5][0-9]|[6][0]))')
+testset1=['15:30','24:00']
+filter(reg1.match,testset1)
+
+ +
+
+
+ +
+
+ + +
+
Out[12]:
+ + + +
+
['15:30']
+
+ +
+ +
+
+ +
+
+
+
+
+
+

2) Genus species names

First letter should be capitalized, then it comes with the ".", then six letters in the lower case.

+
+
+
+
+
+
In [13]:
+
+
+
reg2=re.compile('[A-Z]{1}\.[a-z]{6}')
+testset2=['H.sapien','h.sapien']
+filter(reg2.match,testset2)
+
+ +
+
+
+ +
+
+ + +
+
Out[13]:
+ + + +
+
['H.sapien']
+
+ +
+ +
+
+ +
+
+
+
+
+
+

3) Social security numbers

Three numbers, then "-", then two numbers, then "-", then four numbers

+
+
+
+
+
+
In [14]:
+
+
+
reg3=re.compile('[0-9]{3}\-[0-9]{2}\-[0-9]{4}')
+testset3=['389-05-4771','38-051-41']
+filter(reg3.match,testset3)
+
+ +
+
+
+ +
+
+ + +
+
Out[14]:
+ + + +
+
['389-05-4771']
+
+ +
+ +
+
+ +
+
+
+ + + + + + From 35d0511e5a1edb8deef835bd2fe07d44a01c9437 Mon Sep 17 00:00:00 2001 From: yunluyingying Date: Fri, 17 Nov 2017 14:25:10 -0500 Subject: [PATCH 3/3] Problem 1 --- Problem 1.ipynb | 227 + Problem+1.html | 12046 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 12273 insertions(+) create mode 100644 Problem 1.ipynb create mode 100644 Problem+1.html diff --git a/Problem 1.ipynb b/Problem 1.ipynb new file mode 100644 index 0000000..df82679 --- /dev/null +++ b/Problem 1.ipynb @@ -0,0 +1,227 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Challenge 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2) Plot the data " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "data=pd.read_table('chickwts.txt',sep=',')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.boxplot(by='feed',column='weight',grid=False,figsize=(15,15))\n", + "# I am not sure why somehow there is no figure in Jupyter but I did get a barplot in python. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "## 3) A hypothesis test\n", + "\n", + "### Null hypothesis: the weight of chick fed by soybean has no significant difference with the weight of chick fed by sunflower\n", + "### Alternative hypothesis: the weight of chick fed by soybean has significant difference with the weight of chick fed by sunflower" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4) Hypothesis test" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "le=LabelEncoder()\n", + "le.fit(data['feed'])\n", + "data['Index']=le.transform(data['feed'])\n", + "x=data[(data['feed']=='soybean')|(data['feed']=='sunflower')]['Index']\n", + "y=data[(data['feed']=='soybean')|(data['feed']=='sunflower')]['weight']" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimization terminated successfully.\n", + " Current function value: 145.240592\n", + " Iterations: 85\n", + " Function evaluations: 162\n" + ] + }, + { + "data": { + "text/plain": [ + "array([ 284.49999051, 64.53691755])" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "from scipy.optimize import minimize\n", + "from scipy.stats import norm\n", + "\n", + "# null hypothesis\n", + "def null(p,obs):\n", + " B0=p[0]\n", + " sigma=p[1]\n", + " \n", + " expected=B0\n", + " nll=-1*norm(expected,sigma).logpdf(obs).sum()\n", + " return nll\n", + "\n", + "initialGuess1=np.array([1,1])\n", + "fitNull=minimize(null,initialGuess1,method=\"Nelder-Mead\",options={'disp':True},args=(y))\n", + "fitNull.x" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimization terminated successfully.\n", + " Current function value: 138.469162\n", + " Iterations: 194\n", + " Function evaluations: 360\n" + ] + }, + { + "data": { + "text/plain": [ + "array([-83.52384926, 82.48810462, 49.73945446])" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# alternative hypothesis\n", + "def alter(p,obs1,obs2):\n", + " B0=p[0]\n", + " B1=p[1]\n", + " sigma=p[2]\n", + " \n", + " expected=B0+B1*obs1\n", + " nll=-1*norm(expected,sigma).logpdf(obs2).sum()\n", + " return nll\n", + "\n", + "initialGuess1=np.array([1,1,1])\n", + "fitalter=minimize(alter,initialGuess1,method=\"Nelder-Mead\",options={'disp':True},args=(x,y))\n", + "fitalter.x" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0002331767286918307" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from scipy.stats import chi2\n", + "D=2*(fitNull.fun-fitalter.fun)\n", + "1-chi2.cdf(x=D,df=1) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5) Hypothesis test result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## If we consider alfa=0.05, then p value is smaller than 0.05, we can reject the null hyphothesis and conclude that the weight of chick fed by soybean has significant difference with the weight of chick fed by sunflower. " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Problem+1.html b/Problem+1.html new file mode 100644 index 0000000..c7063b3 --- /dev/null +++ b/Problem+1.html @@ -0,0 +1,12046 @@ + + + +Problem 1 + + + + + + + + + + + + + + + + + + + +
+
+ +
+
+
+
+
+

Challenge 1

+
+
+
+
+
+
+
+
+

2) Plot the data

+
+
+
+
+
+
In [2]:
+
+
+
import pandas as pd
+data=pd.read_table('chickwts.txt',sep=',')
+
+ +
+
+
+ +
+
+
+
In [6]:
+
+
+
data.boxplot(by='feed',column='weight',grid=False,figsize=(15,15))
+# I am not sure why somehow there is no figure in Jupyter but I did get a barplot in python. 
+
+ +
+
+
+ +
+
+ + +
+
Out[6]:
+ + + +
+
<matplotlib.axes._subplots.AxesSubplot at 0x115b369d0>
+
+ +
+ +
+
+ +
+
+
+
+
+
+

3) A hypothesis test

Null hypothesis: the weight of chick fed by soybean has no significant difference with the weight of chick fed by sunflower

Alternative hypothesis: the weight of chick fed by soybean has significant difference with the weight of chick fed by sunflower

+
+
+
+
+
+
+
+
+

4) Hypothesis test

+
+
+
+
+
+
In [31]:
+
+
+
le=LabelEncoder()
+le.fit(data['feed'])
+data['Index']=le.transform(data['feed'])
+x=data[(data['feed']=='soybean')|(data['feed']=='sunflower')]['Index']
+y=data[(data['feed']=='soybean')|(data['feed']=='sunflower')]['weight']
+
+ +
+
+
+ +
+
+
+
In [28]:
+
+
+
import numpy as np
+from scipy.optimize import minimize
+from scipy.stats import norm
+
+# null hypothesis
+def null(p,obs):
+    B0=p[0]
+    sigma=p[1]
+    
+    expected=B0
+    nll=-1*norm(expected,sigma).logpdf(obs).sum()
+    return nll
+
+initialGuess1=np.array([1,1])
+fitNull=minimize(null,initialGuess1,method="Nelder-Mead",options={'disp':True},args=(y))
+fitNull.x
+
+ +
+
+
+ +
+
+ + +
+
+ +
+
Optimization terminated successfully.
+         Current function value: 145.240592
+         Iterations: 85
+         Function evaluations: 162
+
+
+
+ +
+
Out[28]:
+ + + +
+
array([ 284.49999051,   64.53691755])
+
+ +
+ +
+
+ +
+
+
+
In [29]:
+
+
+
# alternative hypothesis
+def alter(p,obs1,obs2):
+    B0=p[0]
+    B1=p[1]
+    sigma=p[2]
+    
+    expected=B0+B1*obs1
+    nll=-1*norm(expected,sigma).logpdf(obs2).sum()
+    return nll
+
+initialGuess1=np.array([1,1,1])
+fitalter=minimize(alter,initialGuess1,method="Nelder-Mead",options={'disp':True},args=(x,y))
+fitalter.x
+
+ +
+
+
+ +
+
+ + +
+
+ +
+
Optimization terminated successfully.
+         Current function value: 138.469162
+         Iterations: 194
+         Function evaluations: 360
+
+
+
+ +
+
Out[29]:
+ + + +
+
array([-83.52384926,  82.48810462,  49.73945446])
+
+ +
+ +
+
+ +
+
+
+
In [30]:
+
+
+
from scipy.stats import chi2
+D=2*(fitNull.fun-fitalter.fun)
+1-chi2.cdf(x=D,df=1) 
+
+ +
+
+
+ +
+
+ + +
+
Out[30]:
+ + + +
+
0.0002331767286918307
+
+ +
+ +
+
+ +
+
+
+
+
+
+

5) Hypothesis test result

+
+
+
+
+
+
+
+
+

If we consider alfa=0.05, then p value is smaller than 0.05, we can reject the null hyphothesis and conclude that the weight of chick fed by soybean has significant difference with the weight of chick fed by sunflower.

+
+
+
+
+
+ + + + + +