Gophers With Hammers

Josh Bleecher Snyder

PayPal

Go was designed with tools in mind. (Rob Pike)

Designed with tools in mind

Simple, regular syntax
Simple semantics
Batteries included

Tools everywhere

go
gofmt, goimports
godoc
go test [-cover] [-race]
go vet
gofix, gofmt -r, eg
oracle
golint
godep

and more...

go command

$ go list -f '{{.Deps}}' bytes
[errors io runtime sync sync/atomic unicode unicode/utf8 unsafe]

gofmt

from

for{
fmt.Println(      "I feel pretty." );
       }

for {
    fmt.Println("I feel pretty.")
}

godoc

$ godoc strings Repeat
func Repeat(s string, count int) string
    Repeat returns a new string consisting of count copies of the string s.

go vet

Oops

if suffix != ".md" || suffix != ".markdown" {

Flagged

suspect or: suffix != ".md" || suffix != ".markdown"

go tool cover -mode=set

func Repeat(s string, count int) string {
    b := make([]byte, len(s)*count)
    bp := 0
    for i := 0; i < count; i++ {
        bp += copy(b[bp:], s)
    }
    return string(b)
}

func Repeat(s string, count int) string {
    GoCover.Count[0] = 1
    b := make([]byte, len(s)*count)
    bp := 0
    for i := 0; i < count; i++ {
        GoCover.Count[2] = 1
        bp += copy(b[bp:], s)
    }
    GoCover.Count[1] = 1
    return string(b)
}

go test -cover

$ go test -coverprofile=c.out strings
ok      strings    0.455s    coverage: 96.9% of statements

$ go tool cover -func=c.out
strings/reader.go:    Len                66.7%
strings/reader.go:    Read                100.0%
strings/reader.go:    ReadAt                100.0%
strings/reader.go:    ReadByte            100.0%
strings/reader.go:    UnreadByte            100.0%
strings/reader.go:    ReadRune            100.0%
strings/reader.go:    UnreadRune            100.0%
strings/reader.go:    Seek                90.9%
strings/reader.go:    WriteTo                83.3%
...

$ go tool cover -html=c.out
# opens a browser window, shows line-by-line coverage

Tools to make tools

text/template
go/build
go/doc
go/format
go/{parser,token,ast,printer}
go.tools/go/types and friends

and more...

impl

Generate implementation stubs given an interface.

go get github.com/josharian/impl

Generate

$ impl 'f *File' io.Reader
func (f *File) Read(p []byte) (n int, err error) {
}

from

package io

type Reader interface {
    Read(p []byte) (n int, err error)
}

impl

Generate

$ impl 'f *File' io.ReadWriter
func (f *File) Read(p []byte) (n int, err error) {
}

func (f *File) Write(p []byte) (n int, err error) {
}

from

package io

type ReadWriter interface {
    Reader
    Writer
}

impl

Generate

$ impl 'c *Ctx' http.Handler
func (c *Ctx) ServeHTTP(http.ResponseWriter, *http.Request) {
}

from

package http

type Handler interface {
    ServeHTTP(ResponseWriter, *Request)
}

Plan

Find import path and interface name

http.Handler ⇒ net/http, Handler

Parse interface

net/http, Handler ⇒ {{"ServeHTTP", {{"", "http.ResponseWriter"}, {"", "*http.Request"}}, {}}}}

Generate output

{{"ServeHTTP", {{"", "http.ResponseWriter"}, {"", "*http.Request"}}, {}}}} ⇒ profit!

goimports ftw

import "golang.org/x/tools/imports"

// +build ignore

package main

import (
	"fmt"

	"golang.org/x/tools/imports"
)

func main() {
    iface := "http.Handler"
    src := "package hack; var i " + iface
    fmt.Println(src, "\n---")

    imp, _ := imports.Process("", []byte(src), nil)
    // ignoring errors throughout this presentation
    fmt.Println(string(imp))
}

Hello, AST

*ast.File {
.  Package: 1:1
.  Name: *ast.Ident {
.  .  NamePos: 1:9
.  .  Name: "hack"
.  }
.  Decls: []ast.Decl (len = 2) {
.  .  0: *ast.GenDecl {
.  .  .  TokPos: 1:15
.  .  .  Tok: import
.  .  .  Lparen: -
.  .  .  Specs: []ast.Spec (len = 1) {
.  .  .  .  0: *ast.ImportSpec {
.  .  .  .  .  Path: *ast.BasicLit {
.  .  .  .  .  .  ValuePos: 1:22
.  .  .  .  .  .  Kind: STRING
.  .  .  .  .  .  Value: "\"net/http\""
.  .  .  .  .  }

[truncated]

Extract the import path

import (
    "go/parser"
    "go/token"
)

// +build ignore

package main

import (
	"fmt"
	"go/parser"
	"go/token"
	"strconv"
)

func main() {
    src := `package hack; import "net/http"; var i http.Handler`

    fset := token.NewFileSet()
    f, _ := parser.ParseFile(fset, "", src, 0)

    raw := f.Imports[0].Path.Value
    path, _ := strconv.Unquote(raw)
    fmt.Println(raw, "\n", path)
}

Extract the interface name

import "go/ast"

// +build ignore

package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
)

func main() {
    src := `package hack; import "net/http"; var i http.Handler`
    f, _ := parser.ParseFile(token.NewFileSet(), "", src, 0)

    decl := f.Decls[1].(*ast.GenDecl)      // var i http.Handler
    spec := decl.Specs[0].(*ast.ValueSpec) // i http.Handler
    sel := spec.Type.(*ast.SelectorExpr)   // http.Handler
    id := sel.Sel.Name                     // Handler
    fmt.Println(id)
}

A GenDecl can have many Specs

var (
    r io.Reader
    w io.Writer
)

Plan

Find import path and interface name

http.Handler ⇒ net/http, Handler

Parse interface

net/http, Handler ⇒ {{"ServeHTTP", {{"", "http.ResponseWriter"}, {"", "*http.Request"}}, {}}}}

Generate output

{{"ServeHTTP", {{"", "http.ResponseWriter"}, {"", "*http.Request"}}, {}}}} ⇒ profit!

Data structures

Represent

Read(p []byte) (n int, err error)

Func{
    Name:   "Read",
    Params: []Param{{Name: "p", Type: "[]byte"}},
    Res: []Param{
        {Name: "n", Type: "int"},
        {Name: "err", Type: "error"},
    },
},

Data structures

type Func struct {
    Name   string
    Params []Param
    Res    []Param
}

type Param struct {
    Name string
    Type string
}

Find the code

import "go/build"

// +build ignore

package main

import (
	"fmt"
	"go/build"
)

func main() {
    pkg, _ := build.Import("net/http", "", 0)
    fmt.Println(pkg.Dir)
    fmt.Println(pkg.GoFiles)
}

Find the interface declaration

import "go/printer"

// +build ignore

package main

import (
	"go/ast"
	"go/build"
	"go/parser"
	"go/printer"
	"go/token"
	"os"
	"path/filepath"
)

func main() {
    fset, files := parsePackage("net/http")
    id := "Handler"

    for _, f := range files {
        for _, decl := range f.Decls {
            decl, ok := decl.(*ast.GenDecl)
            if !ok || decl.Tok != token.TYPE {
                continue
            }
            for _, spec := range decl.Specs {
                spec := spec.(*ast.TypeSpec)
                if spec.Name.Name == id {
                    printer.Fprint(os.Stdout, fset, spec)
                }
            }
        }
    }
}


func parsePackage(path string) (*token.FileSet, []*ast.File) {
	pkg, err := build.Import(path, "", 0)
	if err != nil {
		panic(err)
	}

	fset := token.NewFileSet()
	var files []*ast.File
	for _, file := range pkg.GoFiles {
		f, err := parser.ParseFile(fset, filepath.Join(pkg.Dir, file), nil, 0)
		if err != nil {
			continue
		}
		files = append(files, f)
	}
	return fset, files
}

Extract function names

No name? It's an embedded interface. Recurse.

type ByteScanner interface {
    ByteReader
    UnreadByte() error
}

Extract params and results

No name? Just use "".

type ByteWriter interface {
    WriteByte(c byte) error
}

Qualify types

Types can be arbitrarily complicated.

type CrazyGopher interface {
    CrazyGoph(int) func(chan<- [32]byte, map[string]int64) ([]rune, error)
}

And we need to rewrite some of them.

int ⇒ int
*Request ⇒ *http.Request
io.Reader ⇒ io.Reader
func(io.Reader, chan map[S][]*T) ⇒ func(io.Reader, chan map[foo.S][]*foo.T))

Qualify types

// +build ignore

package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
)

func main() {
    src := `
package http
type Handler interface {
    ServeHTTP(ResponseWriter, *Request)
}
`
    f, _ := parser.ParseFile(token.NewFileSet(), "", src, 0)
    typ := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.TypeSpec).Type.(*ast.InterfaceType)
    fndecl := typ.Methods.List[0].Type.(*ast.FuncType)
    // fndecl: (ResponseWriter, *Request)

    ast.Inspect(fndecl, func(n ast.Node) bool {
        if ident, ok := n.(*ast.Ident); ok {
            fmt.Println(ident.Name)
        }
        return true
    })
}

Plan

Find import path and interface name

http.Handler ⇒ net/http, Handler

Parse interface

net/http, Handler ⇒ {{"ServeHTTP", {{"", "http.ResponseWriter"}, {"", "*http.Request"}}, {}}}}

Generate output

{{"ServeHTTP", {{"", "http.ResponseWriter"}, {"", "*http.Request"}}, {}}}} ⇒ profit!

Method type

type Method struct {
    Recv string
    Func
}

type Func struct {
    Name   string
    Params []Param
    Res    []Param
}

type Param struct {
    Name string
    Type string
}

Use text/template

// +build ignore

package main

import (
	"os"
	"text/template"
)

func main() {
    const stub = "func ({{.Recv}}) {{.Name}}" +
        "({{range .Params}}{{.Name}} {{.Type}}, {{end}})" +
        "({{range .Res}}{{.Name}} {{.Type}}, {{end}})" +
        "{\n}\n\n"
    tmpl := template.Must(template.New("test").Parse(stub))

    m := Method{
        Recv: "f *File",
        Func: Func{
            Name: "Close",
            Res:  []Param{{Type: "error"}},
        },
    }

    tmpl.Execute(os.Stdout, m)
}


// Method represents a method signature.
type Method struct {
	Recv string
	Func
}

// Func represents a function signature.
type Func struct {
	Name   string
	Params []Param
	Res    []Param
}

// Param represents a parameter in a function or method signature.
type Param struct {
	Name string
	Type string
}

Ugly is ok

import "go/format"

// +build ignore

package main

import (
	"fmt"
	"go/format"
)

func main() {
    ugly := `func (f *File) Read(p []byte, )(n int, err error, ){}`
    fmt.Println(ugly)
    pretty, _ := format.Source([]byte(ugly))
    fmt.Println(string(pretty))
}

Great success

Full code plus tests at github.com/josharian/impl

Tips

Use go get -d to download lots of code from godoc.org/-/index. (Don't forget to set a temporary GOPATH!)

Use (and improve) github.com/yuroyoro/goast-viewer.

You don't have to generate all the code. And generating data is even better.

The go/ast docs are your friend.

go.tools/go/types is powerful.

go generate is coming.

Nails!

Break up long strings
Enums and flags to Stringers
Dynamic code analysis
Vet checks
Reflect ⇒ codegen
Convention-based http dispatch
Detect "last line" copy/paste bugs
AST-aware diff, merge, blame; automated fork analysis
Machine learning models of ASTs: anomaly detection, bug-prone code detection
Code fingerprinting
Examine usage patterns
Compiler stress tests

Thank you

Josh Bleecher Snyder

PayPal

josharian@gmail.com

@offbymany

Gophers With Hammers

Go was designed with tools in mind. (Rob Pike)

Designed with tools in mind

Tools everywhere

go command

gofmt

godoc

go vet

go tool cover -mode=set

go test -cover

Tools to make tools

Hammers are fun!

impl

impl

impl

Plan

goimports ftw

Hello, AST

Extract the import path

Extract the interface name

Plan

Data structures

Data structures

Find the code

Find the interface declaration

Extract function names

Extract params and results

Qualify types

Qualify types

Plan

Method type

Use text/template

Ugly is ok

Great success

Tips

Nails!

Thank you