From 4b5e3bd829321e5dd4dd8585ada23458eed318af Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 1 Feb 2020 15:57:46 +0100 Subject: [PATCH 01/11] Add DataStruct method to LasGo that converts data to Struct and returns []interface --- common.go | 232 +++++++++++++++++++++++++++++++++++++++++++++++++ common_test.go | 44 ++++++++++ go.mod | 8 ++ go.sum | 7 ++ las.go | 136 ++++++++--------------------- 5 files changed, 327 insertions(+), 100 deletions(-) create mode 100644 common.go create mode 100644 go.sum diff --git a/common.go b/common.go new file mode 100644 index 0000000..18cb364 --- /dev/null +++ b/common.go @@ -0,0 +1,232 @@ +package lasgo + +import ( + "context" + "errors" + "fmt" + "reflect" + "regexp" + "strings" + + "github.com/mitchellh/mapstructure" +) + +// StructorConfig is used to expose a subset of the configuration options +// provided by the mapstructure package. +// +// See: https://godoc.org/github.com/mitchellh/mapstructure#DecoderConfig +type StructorConfig struct { + + // DecodeHook, if set, will be called before any decoding and any + // type conversion (if WeaklyTypedInput is on). This lets you modify + // the values before they're set down onto the resulting struct. + // + // If an error is returned, the entire decode will fail with that + // error. + DecodeHook mapstructure.DecodeHookFunc + + // If WeaklyTypedInput is true, the decoder will make the following + // "weak" conversions: + // + // - bools to string (true = "1", false = "0") + // - numbers to string (base 10) + // - bools to int/uint (true = 1, false = 0) + // - strings to int/uint (base implied by prefix) + // - int to bool (true if value != 0) + // - string to bool (accepts: 1, t, T, TRUE, true, True, 0, f, F, + // FALSE, false, False. Anything else is an error) + // - empty array = empty map and vice versa + // - negative numbers to overflowed uint values (base 10) + // - slice of maps to a merged map + // - single values are converted to slices if required. Each + // element is weakly decoded. For example: "4" can become []int{4} + // if the target type is an int slice. + // + WeaklyTypedInput bool +} + +// PostUnmarshaler allows you to further modify all results after unmarshaling. +// The ConcreteStruct pointer must implement this interface to make use of this feature. +type PostUnmarshaler interface { + + // PostUnmarshal is called for each row after all results have been fetched. + // You can use it to further modify the values of each ConcreteStruct. + PostUnmarshal(ctx context.Context, row, count int) error +} + +// DataOptions is used to modify the default behavior. +type DataOptions struct { + // ConcreteStruct can be set to any concrete struct (not a pointer). + // When set, the mapstructure package is used to convert the returned + // results automatically from a map to a struct. The `dbq` struct tag + // can be used to map column names to the struct's fields. + // + // See: https://godoc.org/github.com/mitchellh/mapstructure + ConcreteStruct interface{} + + // DecoderConfig is used to configure the decoder used by the mapstructure + // package. If it's not supplied, a default StructorConfig is assumed. This means + // WeaklyTypedInput is set to true and no DecodeHook is provided. + // + // See: https://godoc.org/github.com/mitchellh/mapstructure + DecoderConfig *StructorConfig +} + +// index returns the position of an element in a slice of strings +func index(slice []string, item string) int { + for i := range slice { + if slice[i] == item { + return i + } + } + return -1 +} + +func removeComment(str string) []string { + trimmedStr := strings.TrimSpace(str) + strVec := strings.Split(trimmedStr, "\n") + var result []string + for _, line := range strVec { + fStr := strings.TrimSpace(line) + if !strings.HasPrefix(fStr, "#") && len(fStr) > 0 { + result = append(result, fStr) + } + } + return result +} + +func pattern(str string) *regexp.Regexp { + return regexp.MustCompile(str) +} + +func chunk(s []string, n int) (store [][]string) { + for i := 0; i < len(s); i += n { + if i+n >= len(s) { + store = append(store, s[i:]) + } else { + store = append(store, s[i:i+n]) + } + } + return +} + +// metadata - picks out version and wrap state of the file +func metadata(str string) (version string, wrap bool) { + sB := strings.Split(pattern("~V(?:\\w*\\s*)*\n\\s*").Split(str, 2)[1], "~")[0] + sw := removeComment(sB) + accum := [][]string{} + for _, val := range sw { + current := pattern("\\s{2,}|\\s*:").Split(val, -1)[0:2] + accum = append(accum, current) + } + version = accum[0][1] + if strings.ToLower(accum[1][1]) == "yes" { + wrap = true + } else { + wrap = false + } + return +} + +func property(str string, key string) (property map[string]WellProps, err error) { + err = errors.New("property cannot be found") + property = make(map[string]WellProps) + + regDict := map[string]string{ + "curve": "~C(?:\\w*\\s*)*\\n\\s*", + "param": "~P(?:\\w*\\s*)*\\n\\s*", + "well": "~W(?:\\w*\\s*)*\\n\\s*", + } + prop, ok := regDict[key] + if !ok { + return + } + substr := pattern(prop).Split(str, 2) + var sw []string + if len(substr) > 1 { + sw = removeComment(strings.Split(substr[1], "~")[0]) + } + if len(sw) > 0 { + for _, val := range sw { + root := pattern("\\s*[.]\\s+").ReplaceAllString(val, " none ") + title := pattern("[.]|\\s+").Split(root, 2)[0] + unit := pattern("\\s+").Split(pattern("^\\w+\\s*[.]*s*").Split(root, 2)[1], 2)[0] + desc := strings.TrimSpace(strings.Split(root, ":")[1]) + desc = pattern("\\d+\\s*").ReplaceAllString(desc, "") + if len(desc) < 1 { + desc = "none" + } + vD := pattern("\\s{2,}\\w*\\s{2,}").Split(strings.Split(root, ":")[0], -1) + var value string + if len(vD) > 2 && len(vD[len(vD)-1]) > 0 { + value = strings.TrimSpace(vD[len(vD)-2]) + } else { + value = strings.TrimSpace(vD[len(vD)-1]) + } + property[title] = WellProps{unit, desc, value} + } + return property, nil + } + return +} + +func structConvert(ctx context.Context, vals [][]string, header []string, o *DataOptions) ([]interface{}, error) { + var ( + outStruct = []interface{}{} + ) + + for _, row := range vals { + + // map header to row value + rowMap := map[string]interface{}{} + + if len(header) != len(row) { + return nil, fmt.Errorf("length of each row must be same as length of header") + } + + for idx, field := range row { + headerI := strings.ToLower(header[idx]) + rowMap[headerI] = field + } + + res := reflect.New(reflect.TypeOf(o.ConcreteStruct)).Interface() + if o.DecoderConfig != nil { + dc := &mapstructure.DecoderConfig{ + DecodeHook: o.DecoderConfig.DecodeHook, + ZeroFields: true, + TagName: "las", + WeaklyTypedInput: o.DecoderConfig.WeaklyTypedInput, + Result: res, + } + decoder, err := mapstructure.NewDecoder(dc) + if err != nil { + return nil, err + } + + err = decoder.Decode(rowMap) + if err != nil { + return nil, err + } + + } else { + dc := &mapstructure.DecoderConfig{ + ZeroFields: true, + TagName: "las", + WeaklyTypedInput: true, + Result: res, + } + decoder, err := mapstructure.NewDecoder(dc) + if err != nil { + return nil, err + } + err = decoder.Decode(rowMap) + if err != nil { + return nil, err + } + } + + outStruct = append(outStruct, res) + } + + return outStruct, nil +} diff --git a/common_test.go b/common_test.go index 1b36575..ed6abe8 100644 --- a/common_test.go +++ b/common_test.go @@ -1,8 +1,14 @@ package lasgo import ( + "context" "reflect" "testing" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" + "github.com/mitchellh/mapstructure" ) func TestChunk(t *testing.T) { @@ -24,3 +30,41 @@ func TestRemoveComment(t *testing.T) { } } } + +func TestStructConvert(t *testing.T) { + type list struct { + Index int `las:"index"` + Item string `las:"item"` + } + + testData := []string{"1", "books", "2", "bicycles", "3", "cars", "4", "computers"} + + header := []string{"index", "item"} + + chnk := chunk(testData, len(header)) + + expected := []interface{}{ + &list{Index: int(1), Item: "books"}, + &list{Index: int(2), Item: "bicycles"}, + &list{Index: int(3), Item: "cars"}, + &list{Index: int(4), Item: "computers"}, + } + + opts := &DataOptions{ConcreteStruct: list{}, DecoderConfig: &StructorConfig{ + DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), + WeaklyTypedInput: true}} + + // opts := &DataOptions{ConcreteStruct: list{}} + + ctx := context.Background() + actual, err := structConvert(ctx, chnk, header, opts) + if err != nil { + t.Errorf("Error encountered: %s\n", err) + } + spew.Dump(actual) + + if !cmp.Equal(expected, actual) { + t.Errorf("wrong val: expected: %T %v actual: %T %v\n", expected, expected, actual, actual) + } + +} diff --git a/go.mod b/go.mod index 2db37eb..97571e8 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,11 @@ module github.com/iykekings/las-go go 1.13 + +require ( + github.com/davecgh/go-spew v1.1.1 + github.com/google/go-cmp v0.4.0 + github.com/mitchellh/mapstructure v1.1.2 + golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ccf299c --- /dev/null +++ b/go.sum @@ -0,0 +1,7 @@ +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/las.go b/las.go index e946f94..a7b9eb8 100644 --- a/las.go +++ b/las.go @@ -1,11 +1,10 @@ package lasgo import ( - "errors" + "context" "fmt" "io/ioutil" "os" - "regexp" "strings" ) @@ -15,16 +14,6 @@ type LasType struct { content string } -// index returns the position of an element in a slice of strings -func index(slice []string, item string) int { - for i := range slice { - if slice[i] == item { - return i - } - } - return -1 -} - //Las creates an instance of LasType func Las(path string) (*LasType, error) { bs, err := ioutil.ReadFile(path) @@ -33,34 +22,6 @@ func Las(path string) (*LasType, error) { return &l, err } -func removeComment(str string) []string { - trimmedStr := strings.TrimSpace(str) - strVec := strings.Split(trimmedStr, "\n") - var result []string - for _, line := range strVec { - fStr := strings.TrimSpace(line) - if !strings.HasPrefix(fStr, "#") && len(fStr) > 0 { - result = append(result, fStr) - } - } - return result -} - -func pattern(str string) *regexp.Regexp { - return regexp.MustCompile(str) -} - -func chunk(s []string, n int) (store [][]string) { - for i := 0; i < len(s); i += n { - if i+n >= len(s) { - store = append(store, s[i:]) - } else { - store = append(store, s[i:i+n]) - } - } - return -} - // WellProps contains basic definition of a single well measurement type WellProps struct { unit string @@ -97,6 +58,41 @@ func (l *LasType) Data() [][]string { return chunk(sBs, len(hds)) } +// DataStruct just like Data but returns output in specified struct format +func (l *LasType) DataStruct(opt *DataOptions) []interface{} { + + hds, err := l.Header() + if err != nil { + panic("No data in file") + } + sB := pattern("~A(?:\\w*\\s*)*\n").Split(l.content, 2)[1] + sBs := pattern("\\s+").Split(strings.TrimSpace(sB), -1) + + var ( + o *DataOptions + store [][]string + ) + + if opt != nil { + o = opt + } + + store = chunk(sBs, len(hds)) + + if o != nil { + ctx := context.Background() + + output, err := structConvert(ctx, store, hds, o) + if err != nil { + panic(err) + } + + return output + } + + return nil +} + // Version - returns the version of the las file func (l *LasType) Version() (version string) { version, _ = metadata(l.content) @@ -199,63 +195,3 @@ func (l *LasType) ToCSV(filename string) { file.WriteString(strings.Join(val, ",") + "\n") } } - -// metadata - picks out version and wrap state of the file -func metadata(str string) (version string, wrap bool) { - sB := strings.Split(pattern("~V(?:\\w*\\s*)*\n\\s*").Split(str, 2)[1], "~")[0] - sw := removeComment(sB) - accum := [][]string{} - for _, val := range sw { - current := pattern("\\s{2,}|\\s*:").Split(val, -1)[0:2] - accum = append(accum, current) - } - version = accum[0][1] - if strings.ToLower(accum[1][1]) == "yes" { - wrap = true - } else { - wrap = false - } - return -} - -func property(str string, key string) (property map[string]WellProps, err error) { - err = errors.New("property cannot be found") - property = make(map[string]WellProps) - - regDict := map[string]string{ - "curve": "~C(?:\\w*\\s*)*\\n\\s*", - "param": "~P(?:\\w*\\s*)*\\n\\s*", - "well": "~W(?:\\w*\\s*)*\\n\\s*", - } - prop, ok := regDict[key] - if !ok { - return - } - substr := pattern(prop).Split(str, 2) - var sw []string - if len(substr) > 1 { - sw = removeComment(strings.Split(substr[1], "~")[0]) - } - if len(sw) > 0 { - for _, val := range sw { - root := pattern("\\s*[.]\\s+").ReplaceAllString(val, " none ") - title := pattern("[.]|\\s+").Split(root, 2)[0] - unit := pattern("\\s+").Split(pattern("^\\w+\\s*[.]*s*").Split(root, 2)[1], 2)[0] - desc := strings.TrimSpace(strings.Split(root, ":")[1]) - desc = pattern("\\d+\\s*").ReplaceAllString(desc, "") - if len(desc) < 1 { - desc = "none" - } - vD := pattern("\\s{2,}\\w*\\s{2,}").Split(strings.Split(root, ":")[0], -1) - var value string - if len(vD) > 2 && len(vD[len(vD)-1]) > 0 { - value = strings.TrimSpace(vD[len(vD)-2]) - } else { - value = strings.TrimSpace(vD[len(vD)-1]) - } - property[title] = WellProps{unit, desc, value} - } - return property, nil - } - return -} From 9e0c7b4a7cbb74d40b2d37b25e7f9c73379576b6 Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 1 Feb 2020 16:24:56 +0100 Subject: [PATCH 02/11] Add Code for PostUnmarshaller (very useful) --- common.go | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/common.go b/common.go index 18cb364..e6518b1 100644 --- a/common.go +++ b/common.go @@ -6,9 +6,12 @@ import ( "fmt" "reflect" "regexp" + "runtime" "strings" "github.com/mitchellh/mapstructure" + "golang.org/x/sync/errgroup" + "golang.org/x/xerrors" ) // StructorConfig is used to expose a subset of the configuration options @@ -70,6 +73,9 @@ type DataOptions struct { // // See: https://godoc.org/github.com/mitchellh/mapstructure DecoderConfig *StructorConfig + + // ConcurrentPostUnmarshal can be set to true if PostUnmarshal must be called concurrently. + ConcurrentPostUnmarshal bool } // index returns the position of an element in a slice of strings @@ -225,6 +231,53 @@ func structConvert(ctx context.Context, vals [][]string, header []string, o *Dat } } + if len(outStruct) > 0 { + csTyp := reflect.TypeOf(reflect.New(reflect.TypeOf(o.ConcreteStruct)).Interface()) + ics := reflect.TypeOf((*PostUnmarshaler)(nil)).Elem() + + if csTyp.Implements(ics) { + rows := reflect.ValueOf(outStruct) + count := rows.Len() + + if o.ConcurrentPostUnmarshal && runtime.GOMAXPROCS(0) > 1 { + g, newCtx := errgroup.WithContext(ctx) + + for i := 0; i < count; i++ { + i := i + g.Go(func() error { + if err := newCtx.Err(); err != nil { + return err + } + + row := reflect.ValueOf(rows.Index(i).Interface()) + retVals := row.MethodByName("PostUnmarshal").Call([]reflect.Value{reflect.ValueOf(newCtx), reflect.ValueOf(i), reflect.ValueOf(count)}) + err := retVals[0].Interface() + if err != nil { + return xerrors.Errorf("dbq.PostUnmarshal @ row %d: %w", i, err) + } + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + } else { + for i := 0; i < count; i++ { + if err := ctx.Err(); err != nil { + return nil, err + } + row := reflect.ValueOf(rows.Index(i).Interface()) + retVals := row.MethodByName("lasData.PostUnmarshal").Call([]reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(i), reflect.ValueOf(count)}) + err := retVals[0].Interface() + if err != nil { + return nil, xerrors.Errorf("lasData.PostUnmarshal @ row %d: %w", i, err) + } + } + } + } + } + outStruct = append(outStruct, res) } From 7ace5a29a4d66b05282b08a767b8d932d080352b Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 1 Feb 2020 17:02:14 +0100 Subject: [PATCH 03/11] Add integration test for DataStruct method --- test/integration_test.go | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/test/integration_test.go b/test/integration_test.go index 018d174..3bcff68 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -4,9 +4,21 @@ import ( "fmt" "testing" + "github.com/google/go-cmp/cmp" lasgo "github.com/iykekings/las-go" ) +type dataRow struct { + Dept float64 `las:"DEPT"` + Dt string `las:"DT"` + Rhob float64 `las:"RHOB"` + Nphi string `las:"NPHI"` + Sflu float64 `las:"SFLU"` + Sfla float64 `las:"SFLA"` + Ilm string `las:"ILM"` + Ild float64 `las:"ILD"` +} + func TestRowCount(t *testing.T) { las, err := lasgo.Las("../sample/example1.las") if err != nil { @@ -48,3 +60,62 @@ func TestWrap(t *testing.T) { t.Errorf("las.ColumnCount() == %v, want %v", wrap, false) } } + +func TestDataStruct(t *testing.T) { + las, err := lasgo.Las("../sample/example1.las") + if err != nil { + panic(err) + } + + expected := []interface{}{ + &dataRow{ + Dept: float64(1670), + Dt: string("123.450"), + Rhob: float64(2550), + Nphi: string("0.450"), + Sflu: float64(123.45), + Sfla: float64(123.45), + Ilm: string("110.200"), + Ild: float64(105.6), + }, + &dataRow{ + Dept: float64(1669.875), + Dt: string("123.450"), + Rhob: float64(2550), + Nphi: string("0.450"), + Sflu: float64(123.45), + Sfla: float64(123.45), + Ilm: string("110.200"), + Ild: float64(105.6), + }, + &dataRow{ + Dept: float64(1669.75), + Dt: string("123.450"), + Rhob: float64(2550), + Nphi: string("0.450"), + Sflu: float64(123.45), + Sfla: float64(123.45), + Ilm: string("110.200"), + Ild: float64(105.6), + }, + &dataRow{ + Dept: float64(1669.745), + Dt: string("123.450"), + Rhob: float64(2550), + Nphi: string("-999.25"), + Sflu: float64(123.45), + Sfla: float64(123.45), + Ilm: string("110.200"), + Ild: float64(105.6), + }, + } + + opts := &lasgo.DataOptions{ConcreteStruct: dataRow{}} + + actual := las.DataStruct(opts) + + if !cmp.Equal(expected, actual) { + t.Errorf("wrong val: expected: %T %v actual: %T %v\n", expected, expected, actual, actual) + } + +} From 6d29804bbcb90080a454d8add843771e1b31e734 Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 1 Feb 2020 18:09:09 +0100 Subject: [PATCH 04/11] Fix and Add Integration code for PostUnmarshal --- common.go | 77 +++++++++++++------------- test/integration_test.go | 114 +++++++++++++++++++++++++++++++++++---- 2 files changed, 142 insertions(+), 49 deletions(-) diff --git a/common.go b/common.go index e6518b1..a3f2463 100644 --- a/common.go +++ b/common.go @@ -231,54 +231,55 @@ func structConvert(ctx context.Context, vals [][]string, header []string, o *Dat } } - if len(outStruct) > 0 { - csTyp := reflect.TypeOf(reflect.New(reflect.TypeOf(o.ConcreteStruct)).Interface()) - ics := reflect.TypeOf((*PostUnmarshaler)(nil)).Elem() - - if csTyp.Implements(ics) { - rows := reflect.ValueOf(outStruct) - count := rows.Len() - - if o.ConcurrentPostUnmarshal && runtime.GOMAXPROCS(0) > 1 { - g, newCtx := errgroup.WithContext(ctx) - - for i := 0; i < count; i++ { - i := i - g.Go(func() error { - if err := newCtx.Err(); err != nil { - return err - } - - row := reflect.ValueOf(rows.Index(i).Interface()) - retVals := row.MethodByName("PostUnmarshal").Call([]reflect.Value{reflect.ValueOf(newCtx), reflect.ValueOf(i), reflect.ValueOf(count)}) - err := retVals[0].Interface() - if err != nil { - return xerrors.Errorf("dbq.PostUnmarshal @ row %d: %w", i, err) - } - return nil - }) - } + outStruct = append(outStruct, res) + } - if err := g.Wait(); err != nil { - return nil, err - } - } else { - for i := 0; i < count; i++ { - if err := ctx.Err(); err != nil { - return nil, err + // PostUnmarshal code + if len(outStruct) > 0 { + csTyp := reflect.TypeOf(reflect.New(reflect.TypeOf(o.ConcreteStruct)).Interface()) + ics := reflect.TypeOf((*PostUnmarshaler)(nil)).Elem() + + if csTyp.Implements(ics) { + rows := reflect.ValueOf(outStruct) + count := rows.Len() + + if o.ConcurrentPostUnmarshal && runtime.GOMAXPROCS(0) > 1 { + g, newCtx := errgroup.WithContext(ctx) + + for i := 0; i < count; i++ { + i := i + g.Go(func() error { + if err := newCtx.Err(); err != nil { + return err } + row := reflect.ValueOf(rows.Index(i).Interface()) - retVals := row.MethodByName("lasData.PostUnmarshal").Call([]reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(i), reflect.ValueOf(count)}) + retVals := row.MethodByName("PostUnmarshal").Call([]reflect.Value{reflect.ValueOf(newCtx), reflect.ValueOf(i), reflect.ValueOf(count)}) err := retVals[0].Interface() if err != nil { - return nil, xerrors.Errorf("lasData.PostUnmarshal @ row %d: %w", i, err) + return xerrors.Errorf("lasData.PostUnmarshal @ row %d: %w", i, err) } + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + } else { + for i := 0; i < count; i++ { + if err := ctx.Err(); err != nil { + return nil, err + } + row := reflect.ValueOf(rows.Index(i).Interface()) + retVals := row.MethodByName("lasData.PostUnmarshal").Call([]reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(i), reflect.ValueOf(count)}) + err := retVals[0].Interface() + if err != nil { + return nil, xerrors.Errorf("lasData.PostUnmarshal @ row %d: %w", i, err) } } } } - - outStruct = append(outStruct, res) } return outStruct, nil diff --git a/test/integration_test.go b/test/integration_test.go index 3bcff68..1827fbe 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -1,24 +1,15 @@ package test import ( + "context" "fmt" "testing" + "github.com/davecgh/go-spew/spew" "github.com/google/go-cmp/cmp" lasgo "github.com/iykekings/las-go" ) -type dataRow struct { - Dept float64 `las:"DEPT"` - Dt string `las:"DT"` - Rhob float64 `las:"RHOB"` - Nphi string `las:"NPHI"` - Sflu float64 `las:"SFLU"` - Sfla float64 `las:"SFLA"` - Ilm string `las:"ILM"` - Ild float64 `las:"ILD"` -} - func TestRowCount(t *testing.T) { las, err := lasgo.Las("../sample/example1.las") if err != nil { @@ -61,6 +52,17 @@ func TestWrap(t *testing.T) { } } +type dataRow struct { + Dept float64 `las:"DEPT"` + Dt string `las:"DT"` + Rhob float64 `las:"RHOB"` + Nphi string `las:"NPHI"` + Sflu float64 `las:"SFLU"` + Sfla float64 `las:"SFLA"` + Ilm string `las:"ILM"` + Ild float64 `las:"ILD"` +} + func TestDataStruct(t *testing.T) { las, err := lasgo.Las("../sample/example1.las") if err != nil { @@ -119,3 +121,93 @@ func TestDataStruct(t *testing.T) { } } + +type dataRow2 struct { + Dept float64 `las:"DEPT"` + Dt string `las:"DT"` + Rhob float64 `las:"RHOB"` + Nphi string `las:"NPHI"` + Sflu float64 `las:"SFLU"` + Sfla float64 `las:"SFLA"` + Ilm string `las:"ILM"` + Ild float64 `las:"ILD"` +} + +// PostUnmarshaler allows you to further modify all results after unmarshaling. +// The ConcreteStruct pointer must implement this interface to make use of this feature. +func (d *dataRow2) PostUnmarshal(ctx context.Context, row, count int) error { + + // change value of column Rhob to 5505.06 + d.Rhob = float64(5505.06) + + // you can perform many other data manipulation you want in this method + // it is called on every row + + return nil +} + +func TestDataStructPostUnmarshal(t *testing.T) { + las, err := lasgo.Las("../sample/example1.las") + if err != nil { + panic(err) + } + + expected := []interface{}{ + &dataRow2{ + Dept: float64(1670), + Dt: string("123.450"), + Rhob: float64(5505.06), + Nphi: string("0.450"), + Sflu: float64(123.45), + Sfla: float64(123.45), + Ilm: string("110.200"), + Ild: float64(105.6), + }, + &dataRow2{ + Dept: float64(1669.875), + Dt: string("123.450"), + Rhob: float64(5505.06), + Nphi: string("0.450"), + Sflu: float64(123.45), + Sfla: float64(123.45), + Ilm: string("110.200"), + Ild: float64(105.6), + }, + &dataRow2{ + Dept: float64(1669.75), + Dt: string("123.450"), + Rhob: float64(5505.06), + Nphi: string("0.450"), + Sflu: float64(123.45), + Sfla: float64(123.45), + Ilm: string("110.200"), + Ild: float64(105.6), + }, + &dataRow2{ + Dept: float64(1669.745), + Dt: string("123.450"), + Rhob: float64(5505.06), + Nphi: string("-999.25"), + Sflu: float64(123.45), + Sfla: float64(123.45), + Ilm: string("110.200"), + Ild: float64(105.6), + }, + } + + opts := &lasgo.DataOptions{ + ConcreteStruct: dataRow2{}, + // This allows postunmarshal to use + // multiple cpu core (if available) to run concurrently + ConcurrentPostUnmarshal: true, + } + + actual := las.DataStruct(opts) + + spew.Dump(actual) + + if !cmp.Equal(expected, actual) { + t.Errorf("wrong val: expected: %T %+v actual: %T %v\n", expected, expected, actual, actual) + } + +} From 80fc8e5a33faced8a0413113886539e6ab380173 Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 1 Feb 2020 18:16:49 +0100 Subject: [PATCH 05/11] Fix and Add Integration Test code for PostUnmarshal --- common.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.go b/common.go index a3f2463..713fea0 100644 --- a/common.go +++ b/common.go @@ -272,7 +272,7 @@ func structConvert(ctx context.Context, vals [][]string, header []string, o *Dat return nil, err } row := reflect.ValueOf(rows.Index(i).Interface()) - retVals := row.MethodByName("lasData.PostUnmarshal").Call([]reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(i), reflect.ValueOf(count)}) + retVals := row.MethodByName("PostUnmarshal").Call([]reflect.Value{reflect.ValueOf(ctx), reflect.ValueOf(i), reflect.ValueOf(count)}) err := retVals[0].Interface() if err != nil { return nil, xerrors.Errorf("lasData.PostUnmarshal @ row %d: %w", i, err) From d66bcdc5b321193d7d826893c80f83c72a074dbb Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Tue, 4 Feb 2020 19:56:47 +0100 Subject: [PATCH 06/11] Modify DataStruct to use l.Data --- common.go | 4 ++-- common_test.go | 2 +- las.go | 16 +++++++--------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/common.go b/common.go index 713fea0..53d24e7 100644 --- a/common.go +++ b/common.go @@ -176,12 +176,12 @@ func property(str string, key string) (property map[string]WellProps, err error) return } -func structConvert(ctx context.Context, vals [][]string, header []string, o *DataOptions) ([]interface{}, error) { +func structConvert(ctx context.Context, vals *[][]string, header []string, o *DataOptions) ([]interface{}, error) { var ( outStruct = []interface{}{} ) - for _, row := range vals { + for _, row := range *vals { // map header to row value rowMap := map[string]interface{}{} diff --git a/common_test.go b/common_test.go index ed6abe8..39f3185 100644 --- a/common_test.go +++ b/common_test.go @@ -57,7 +57,7 @@ func TestStructConvert(t *testing.T) { // opts := &DataOptions{ConcreteStruct: list{}} ctx := context.Background() - actual, err := structConvert(ctx, chnk, header, opts) + actual, err := structConvert(ctx, &chnk, header, opts) if err != nil { t.Errorf("Error encountered: %s\n", err) } diff --git a/las.go b/las.go index a7b9eb8..5ec918c 100644 --- a/las.go +++ b/las.go @@ -61,13 +61,6 @@ func (l *LasType) Data() [][]string { // DataStruct just like Data but returns output in specified struct format func (l *LasType) DataStruct(opt *DataOptions) []interface{} { - hds, err := l.Header() - if err != nil { - panic("No data in file") - } - sB := pattern("~A(?:\\w*\\s*)*\n").Split(l.content, 2)[1] - sBs := pattern("\\s+").Split(strings.TrimSpace(sB), -1) - var ( o *DataOptions store [][]string @@ -77,12 +70,17 @@ func (l *LasType) DataStruct(opt *DataOptions) []interface{} { o = opt } - store = chunk(sBs, len(hds)) + hds, err := l.Header() + if err != nil { + panic("No data in file") + } + + store = l.Data() if o != nil { ctx := context.Background() - output, err := structConvert(ctx, store, hds, o) + output, err := structConvert(ctx, &store, hds, o) if err != nil { panic(err) } From 481a95b383b64672ac8f8790c50d4c44ff9e3341 Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 8 Feb 2020 15:32:25 +0100 Subject: [PATCH 07/11] Add Docs for Datastruct in ReadMe --- README.md | 82 ++++++++++++++++++++++++++++++++++++++++ test/integration_test.go | 1 + 2 files changed, 83 insertions(+) diff --git a/README.md b/README.md index 72dab94..a71a009 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,88 @@ } ``` +- Read data into a Defined Struct + > Use `Las.DataStruct()` to read your data and have it converted into a defined concrete struct type containing the reading of each log + ```go + import ( + "github.com/davecgh/go-spew/spew" + lasgo "github.com/iykekings/las-go" + ) + + // define a datasruct to represent each data row + type dataRow struct { + Dept float64 `las:"DEPT"` + Dt string `las:"DT"` + Rhob float64 `las:"RHOB"` + Nphi string `las:"NPHI"` + Sflu float64 `las:"SFLU"` + Sfla float64 `las:"SFLA"` + Ilm string `las:"ILM"` + Ild float64 `las:"ILD"` + } + + func main() { + las, err := lasgo.Las("../sample/example1.las") + if err != nil { + panic(err) + } + + opts := &lasgo.DataOptions{ + ConcreteStruct: dataRow{} // pass in datastruct here + } + + spew.Dump(las.DataStruct(opts)) + } + + /** + ([]interface {}) (len=4 cap=4) { + (*main.dataRow)(0xc000093aa0)({ + Dept: (float64) 1670, + Dt: (string) (len=7) "123.450", + Rhob: (float64) 2550, + Nphi: (string) (len=5) "0.450", + Sflu: (float64) 123.45, + Sfla: (float64) 123.45, + Ilm: (string) (len=7) "110.200", + Ild: (float64) 105.6 + }), + (*main.dataRow)(0xc000093b60)({ + Dept: (float64) 1669.875, + Dt: (string) (len=7) "123.450", + Rhob: (float64) 2550, + Nphi: (string) (len=5) "0.450", + Sflu: (float64) 123.45, + Sfla: (float64) 123.45, + Ilm: (string) (len=7) "110.200", + Ild: (float64) 105.6 + }), + (*main.dataRow)(0xc000093c20)({ + Dept: (float64) 1669.75, + Dt: (string) (len=7) "123.450", + Rhob: (float64) 2550, + Nphi: (string) (len=5) "0.450", + Sflu: (float64) 123.45, + Sfla: (float64) 123.45, + Ilm: (string) (len=7) "110.200", + Ild: (float64) 105.6 + }), + (*main.dataRow)(0xc000093ce0)({ + Dept: (float64) 1669.745, + Dt: (string) (len=7) "123.450", + Rhob: (float64) 2550, + Nphi: (string) (len=7) "-999.25", + Sflu: (float64) 123.45, + Sfla: (float64) 123.45, + Ilm: (string) (len=7) "110.200", + Ild: (float64) 105.6 + }) + } + + */ + + ``` + Note: This is just a basic example usage of `las.DataStruct()`. please refer to the test to see a more advanced example of how DataStruct can be used. + - Get the log headers diff --git a/test/integration_test.go b/test/integration_test.go index 1827fbe..3a8cead 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -115,6 +115,7 @@ func TestDataStruct(t *testing.T) { opts := &lasgo.DataOptions{ConcreteStruct: dataRow{}} actual := las.DataStruct(opts) + spew.Dump(actual) if !cmp.Equal(expected, actual) { t.Errorf("wrong val: expected: %T %v actual: %T %v\n", expected, expected, actual, actual) From 5f1a7c0ac4e5d7f633b2297e1e26e99bda5be4de Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 8 Feb 2020 15:45:45 +0100 Subject: [PATCH 08/11] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a71a009..a88ae14 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ */ ``` - Note: This is just a basic example usage of `las.DataStruct()`. please refer to the test to see a more advanced example of how DataStruct can be used. + Note: This is just a basic example usage of `las.DataStruct()`. please refer to the test to see a more advanced example of how DataStruct can be used with the added `PostUnmarshaller` feature. - Get the log headers From 79abfe5982456d3793e626ffe5cc54616180340b Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 8 Feb 2020 17:23:16 +0100 Subject: [PATCH 09/11] Add Usage of DecoderConfig in DataOptions to Doc --- README.md | 23 +++++++++++++++++++++++ common_test.go | 10 +++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a71a009..8d5ac26 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,29 @@ ``` Note: This is just a basic example usage of `las.DataStruct()`. please refer to the test to see a more advanced example of how DataStruct can be used. + If you are try to convert into a data struct that uses `time.Time` type you may encounter an error panic. In such scenario you need to explicitly specify `DecoderConfig` in `Las.DataOptions`. + ```go + + import ( + "github.com/mitchellh/mapstructure" + lasgo "github.com/iykekings/las-go" + ) + + type list struct { + Index int `las:"index"` + Item string `las:"item"` + DateAdded time.Time `las:"date_added"` + } + + opts := &lasgo.DataOptions{ + ConcreteStruct: list{}, + DecoderConfig: &StructorConfig{ + DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), + WeaklyTypedInput: true, + }, + } + ``` + - Get the log headers diff --git a/common_test.go b/common_test.go index 39f3185..f9663c8 100644 --- a/common_test.go +++ b/common_test.go @@ -50,9 +50,13 @@ func TestStructConvert(t *testing.T) { &list{Index: int(4), Item: "computers"}, } - opts := &DataOptions{ConcreteStruct: list{}, DecoderConfig: &StructorConfig{ - DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), - WeaklyTypedInput: true}} + opts := &DataOptions{ + ConcreteStruct: list{}, + DecoderConfig: &StructorConfig{ + DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), + WeaklyTypedInput: true, + }, + } // opts := &DataOptions{ConcreteStruct: list{}} From b463c089e2d0dbd07ac4769053118eaeef127ac4 Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 8 Feb 2020 17:30:43 +0100 Subject: [PATCH 10/11] Update README.md --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 6874c37..321802c 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ - Read data into a Defined Struct > Use `Las.DataStruct()` to read your data and have it converted into a defined concrete struct type containing the reading of each log + ```go import ( "github.com/davecgh/go-spew/spew" @@ -140,18 +141,18 @@ ) type list struct { - Index int `las:"index"` - Item string `las:"item"` - DateAdded time.Time `las:"date_added"` + Index int `las:"index"` + Item string `las:"item"` + DateAdded time.Time `las:"date_added"` } opts := &lasgo.DataOptions{ - ConcreteStruct: list{}, - DecoderConfig: &StructorConfig{ - DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), - WeaklyTypedInput: true, - }, - } + ConcreteStruct: list{}, + DecoderConfig: &StructorConfig{ + DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339), + WeaklyTypedInput: true, + }, + } ``` - Get the log headers From de34c8efd7dfe34f4ec9fec0fb7c3f967ebde55e Mon Sep 17 00:00:00 2001 From: Samuel Ameh Date: Sat, 8 Feb 2020 17:33:54 +0100 Subject: [PATCH 11/11] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 321802c..cab9969 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,8 @@ type list struct { Index int `las:"index"` - Item string `las:"item"` - DateAdded time.Time `las:"date_added"` + Item string `las:"item"` + DateAdded time.Time `las:"date_added"` } opts := &lasgo.DataOptions{