-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathexample_test.go
More file actions
209 lines (196 loc) · 6.4 KB
/
example_test.go
File metadata and controls
209 lines (196 loc) · 6.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
package evalue_test
import (
"fmt"
"github.com/fumin/evalue"
)
func Example() {
// Prepare data.
// Data is from the Moral Typecasting study of the Many Labs 2 project (Klein et al., 2018).
// For simplicity, data is further filtered to contain only those collected from Carleton University, Ottawa, Canada.
//
// Klein RA, Vianello M, Hasselman F, et al. Many Labs 2: Investigating Variation in Replicability Across Samples and Settings. Advances in Methods and Practices in Psychological Science. 2018;1(4):443-490. doi:10.1177/2515245918810225
data := getData()
// Design the experiment.
// The null hypothesis is that the difference between the mean of the two groups is zero.
// The statistical test to be performed is an e-value based two sample t-test.
// Set the significance level at the usual 5%.
alpha := 0.05
// Create an e-process that is tuned to reject the null hypothesis at the fastest rate.
// delta is the minimal clinically relevant standardized effect size.
delta := 0.5176537
eProcess := evalue.NewMom(delta)
// Perform the e-value based test while the experiment is running.
// In contrast to p-values, e-values control the Type I error at all times, and thus allow optional stopping.
stoppingTime := -1
for n := range data {
// Prepare group data for the two sample test.
group1, group2 := splitGroups(data[:n])
if !(len(group1) > 1 && len(group2) > 1) {
continue
}
// Perform the e-value based test with optional stopping.
eValue := eProcess.EValue(group1, group2)
if eValue > 1./alpha {
stoppingTime = n
break
}
}
needed := 100 * float64(stoppingTime) / float64(len(data))
fmt.Printf("Null hypothesis rejected with only %.0f%% (%d/%d) of the data needed.\n", needed, stoppingTime, len(data))
// Output:
// Null hypothesis rejected with only 25% (30/121) of the data needed.
}
func ExampleGetNPlan() {
// alpha is our desired significance level.
alpha := 0.05
// power is our desired statistical power.
power := 0.8
// deltaMin is a lower bound of the true effect size based on domain knowledge.
deltaMin := 0.5
nplan := evalue.GetNPlan(alpha, 1-power, deltaMin)
fmt.Printf("We should plan for a sample size of %d, which would achieve both our desired statistical power and significance level.\n", nplan.N)
fmt.Printf("The good news is we can early-stop our experiment with only %d samples on average, which is not allowed with conventional t-tests.\n", nplan.Mean)
// Output:
// We should plan for a sample size of 102, which would achieve both our desired statistical power and significance level.
// The good news is we can early-stop our experiment with only 61 samples on average, which is not allowed with conventional t-tests.
}
type datum struct {
group int
value float64
}
func splitGroups(data []datum) ([]float64, []float64) {
var g1, g2 []float64
for _, d := range data {
if d.group == 1 {
g1 = append(g1, d.value)
} else {
g2 = append(g2, d.value)
}
}
return g1, g2
}
func getData() []datum {
return []datum{
datum{group: 1, value: 2},
datum{group: 1, value: 3},
datum{group: 1, value: 7},
datum{group: 1, value: 5},
datum{group: 1, value: 6},
datum{group: 2, value: 4},
datum{group: 1, value: 7},
datum{group: 1, value: 3},
datum{group: 2, value: 1},
datum{group: 1, value: 6},
datum{group: 1, value: 7},
datum{group: 1, value: 2},
datum{group: 1, value: 7},
datum{group: 2, value: 5},
datum{group: 1, value: 7},
datum{group: 2, value: 5},
datum{group: 2, value: 1},
datum{group: 1, value: 6},
datum{group: 1, value: 6},
datum{group: 1, value: 7},
datum{group: 1, value: 7},
datum{group: 2, value: 2},
datum{group: 2, value: 3},
datum{group: 2, value: 4},
datum{group: 2, value: 1},
datum{group: 2, value: 5},
datum{group: 1, value: 4},
datum{group: 2, value: 5},
datum{group: 1, value: 6},
datum{group: 2, value: 2},
datum{group: 1, value: 7},
datum{group: 2, value: 5},
datum{group: 1, value: 5},
datum{group: 1, value: 7},
datum{group: 2, value: 1},
datum{group: 2, value: 5},
datum{group: 1, value: 6},
datum{group: 2, value: 7},
datum{group: 2, value: 4},
datum{group: 2, value: 7},
datum{group: 1, value: 6},
datum{group: 2, value: 3},
datum{group: 2, value: 5},
datum{group: 2, value: 3},
datum{group: 1, value: 6},
datum{group: 2, value: 5},
datum{group: 1, value: 3},
datum{group: 2, value: 7},
datum{group: 1, value: 7},
datum{group: 2, value: 7},
datum{group: 2, value: 6},
datum{group: 2, value: 5},
datum{group: 1, value: 7},
datum{group: 1, value: 5},
datum{group: 2, value: 5},
datum{group: 2, value: 5},
datum{group: 2, value: 5},
datum{group: 2, value: 5},
datum{group: 2, value: 3},
datum{group: 2, value: 5},
datum{group: 2, value: 4},
datum{group: 1, value: 5},
datum{group: 1, value: 5},
datum{group: 1, value: 6},
datum{group: 2, value: 2},
datum{group: 1, value: 7},
datum{group: 2, value: 2},
datum{group: 1, value: 6},
datum{group: 1, value: 5},
datum{group: 1, value: 5},
datum{group: 2, value: 1},
datum{group: 1, value: 3},
datum{group: 2, value: 2},
datum{group: 2, value: 5},
datum{group: 1, value: 6},
datum{group: 2, value: 1},
datum{group: 1, value: 6},
datum{group: 1, value: 7},
datum{group: 1, value: 5},
datum{group: 1, value: 4},
datum{group: 1, value: 7},
datum{group: 2, value: 7},
datum{group: 1, value: 7},
datum{group: 1, value: 7},
datum{group: 1, value: 7},
datum{group: 2, value: 5},
datum{group: 1, value: 6},
datum{group: 2, value: 4},
datum{group: 1, value: 6},
datum{group: 2, value: 3},
datum{group: 1, value: 7},
datum{group: 1, value: 5},
datum{group: 1, value: 7},
datum{group: 1, value: 7},
datum{group: 1, value: 7},
datum{group: 1, value: 4},
datum{group: 2, value: 5},
datum{group: 1, value: 6},
datum{group: 1, value: 6},
datum{group: 2, value: 2},
datum{group: 2, value: 3},
datum{group: 1, value: 3},
datum{group: 1, value: 6},
datum{group: 2, value: 2},
datum{group: 2, value: 1},
datum{group: 1, value: 5},
datum{group: 1, value: 7},
datum{group: 1, value: 7},
datum{group: 2, value: 4},
datum{group: 2, value: 5},
datum{group: 2, value: 3},
datum{group: 2, value: 6},
datum{group: 1, value: 5},
datum{group: 1, value: 4},
datum{group: 1, value: 5},
datum{group: 2, value: 5},
datum{group: 1, value: 7},
datum{group: 2, value: 5},
datum{group: 1, value: 3},
datum{group: 1, value: 5},
datum{group: 2, value: 5},
}
}