detect duplicate in JSON String Golang

Use the json.Decoder to walk through the JSON. When an object is found, walk through keys and values checking for duplicate keys.

func check(d *json.Decoder, path []string, dup func(path []string) error) error { // Get next token from JSON t, err := d.Token() if err != nil { return err } // Is it a delimiter? delim, ok := t.(json.Delim) // No, nothing more to check. if !ok { // scaler type, nothing to do return nil } switch delim { case '{': keys := make(map[string]bool) for d.More() { // Get field key. t, err := d.Token() if err != nil { return err } key := t.(string) // Check for duplicates. if keys[key] { // Duplicate found. Call the application's dup function. The // function can record the duplicate or return an error to stop // the walk through the document. if err := dup(append(path, key)); err != nil { return err } } keys[key] = true // Check value. if err := check(d, append(path, key), dup); err != nil { return err } } // consume trailing } if _, err := d.Token(); err != nil { return err } case '[': i := 0 for d.More() { if err := check(d, append(path, strconv.Itoa(i)), dup); err != nil { return err } i++ } // consume trailing ] if _, err := d.Token(); err != nil { return err } } return nil } 

Here's how to call it:

func printDup(path []string) error { fmt.Printf("Duplicate %s\n", strings.Join(path, "/")) return nil } ... data := `{"a": "b", "a":true,"c":["field_3 string 1","field3 string2"], "d": {"e": 1, "e": 2}}` if err := check(json.NewDecoder(strings.NewReader(data)), nil, printDup); err != nil { log.Fatal(err) } 

The output is:

Duplicate a Duplicate d/e 

Run it on the Playground

Here's how to generate an error on the first duplicate key:

var ErrDuplicate = errors.New("duplicate") func dupErr(path []string) error { return ErrDuplicate } ... data := `{"a": "b", "a":true,"c":["field_3 string 1","field3 string2"], "d": {"e": 1, "e": 2}}` err := check(json.NewDecoder(strings.NewReader(data)), nil, dupErr) if err == ErrDuplicate { fmt.Println("found a duplicate") } else if err != nil { // some other error log.Fatal(err) } 

One that would probably work well would be to simply decode, reencode, then check the length of the new json against the old json:

https://play.golang.org/p/50P-x1fxCzp

package main import ( "encoding/json" "fmt" ) func main() { jsn := []byte("{\"a\": \"b\", \"a\":true,\"c\":[\"field_3 string 1\",\"field3 string2\"]}") var m map[string]interface{} err := json.Unmarshal(jsn, &m) if err != nil { panic(err) } l := len(jsn) jsn, err = json.Marshal(m) if err != nil { panic(err) } if l != len(jsn) { panic(fmt.Sprintf("%s: %d (%d)", "duplicate key", l, len(jsn))) } } 

The right way to do it would be to re-implement the json.Decode function, and store a map of keys found, but the above should work (especially if you first stripped any spaces from the json using jsn = bytes.Replace(jsn, []byte(" "), []byte(""), -1) to guard against false positives.