From 0874dc3613b8fdbf106de460017facc23372d736 Mon Sep 17 00:00:00 2001 From: Giacomo Tesio Date: Tue, 22 Dec 2015 10:10:07 +0000 Subject: [PATCH] initial commit --- QA.sh | 39 ++ bin/.gitignore | 2 + buildtools.sh | 10 + continuous-build.sh | 22 + coverity-scan.sh | 14 + devshell.sh | 6 + nvram | Bin 0 -> 512 bytes runOver9P.sh | 38 ++ src/jehanne/cmd/build/build.go | 541 +++++++++++++++++++++++ src/jehanne/cmd/build/codegen.go | 196 ++++++++ src/jehanne/cmd/build/doc.go | 184 ++++++++ src/jehanne/cmd/build/paths.go | 31 ++ src/jehanne/cmd/build/paths_harvey.go | 17 + src/jehanne/cmd/convert/convert.go | 155 +++++++ src/jehanne/cmd/data2c/data2c.go | 39 ++ src/jehanne/cmd/elf2c/elf2c.go | 113 +++++ src/jehanne/cmd/jsonpretty/jsonpretty.go | 53 +++ src/jehanne/cmd/kpdump/kpdump.go | 115 +++++ src/jehanne/cmd/mksys/mksys.go | 325 ++++++++++++++ src/jehanne/cmd/op2/op2.go | 107 +++++ src/jehanne/cmd/preen/.gitignore | 1 + src/jehanne/cmd/preen/preen.go | 157 +++++++ src/jehanne/cmd/profile/profile.go | 131 ++++++ src/jehanne/cmd/removemach/removemach.go | 78 ++++ src/jehanne/cmd/runqemu/runqemu.go | 119 +++++ src/jehanne/cmd/telnet/telnet.go | 244 ++++++++++ src/jehanne/cmd/vendor/doc.go | 45 ++ src/jehanne/cmd/vendor/vendor.go | 248 +++++++++++ 28 files changed, 3030 insertions(+) create mode 100755 QA.sh create mode 100644 bin/.gitignore create mode 100755 buildtools.sh create mode 100755 continuous-build.sh create mode 100755 coverity-scan.sh create mode 100755 devshell.sh create mode 100644 nvram create mode 100755 runOver9P.sh create mode 100644 src/jehanne/cmd/build/build.go create mode 100644 src/jehanne/cmd/build/codegen.go create mode 100644 src/jehanne/cmd/build/doc.go create mode 100644 src/jehanne/cmd/build/paths.go create mode 100644 src/jehanne/cmd/build/paths_harvey.go create mode 100644 src/jehanne/cmd/convert/convert.go create mode 100644 src/jehanne/cmd/data2c/data2c.go create mode 100644 src/jehanne/cmd/elf2c/elf2c.go create mode 100644 src/jehanne/cmd/jsonpretty/jsonpretty.go create mode 100644 src/jehanne/cmd/kpdump/kpdump.go create mode 100644 src/jehanne/cmd/mksys/mksys.go create mode 100644 src/jehanne/cmd/op2/op2.go create mode 100644 src/jehanne/cmd/preen/.gitignore create mode 100644 src/jehanne/cmd/preen/preen.go create mode 100644 src/jehanne/cmd/profile/profile.go create mode 100644 src/jehanne/cmd/removemach/removemach.go create mode 100644 src/jehanne/cmd/runqemu/runqemu.go create mode 100644 src/jehanne/cmd/telnet/telnet.go create mode 100644 src/jehanne/cmd/vendor/doc.go create mode 100644 src/jehanne/cmd/vendor/vendor.go diff --git a/QA.sh b/QA.sh new file mode 100755 index 0000000..89554eb --- /dev/null +++ b/QA.sh @@ -0,0 +1,39 @@ +#!/bin/bash +trap : 2 + +$JEHANNE/hacking/bin/ufs -root=$JEHANNE & +ufspid=$! + +export machineflag=pc +if [ "$(uname)" = "Linux" ] && [ -e /dev/kvm ]; then + export kvmflag='-enable-kvm' + export machineflag='pc,accel=kvm' + if [ ! -w /dev/kvm ]; then + # we don't have access as a regular user + export kvmdo=sudo + fi +fi + +cd $JEHANNE/arch/$ARCH/kern/ +read -r cmd < 0 { + for _, i := range b.SourceFilesCmd { + cmd := exec.Command(tools["cc"], append(args, i)...) + run(b, *shellhack, cmd) + } + return + } + args = append(args, b.SourceFiles...) + cmd := exec.Command(tools["cc"], args...) + run(b, *shellhack, cmd) +} + +func link(b *build) { + log.Printf("Linking %s\n", b.name) + if len(b.SourceFilesCmd) > 0 { + for _, n := range b.SourceFilesCmd { + // Split off the last element of the file + var ext = filepath.Ext(n) + if len(ext) == 0 { + log.Fatalf("refusing to overwrite extension-less source file %v", n) + continue + } + n = n[:len(n)-len(ext)] + f := path.Base(n) + o := f[:len(f)] + ".o" + args := []string{"-o", n, o} + args = append(args, b.Oflags...) + args = append(args, "-L", fromRoot("/arch/$ARCH/lib")) + args = append(args, b.Libs...) + run(b, *shellhack, exec.Command(tools["ld"], args...)) + } + return + } + args := []string{"-o", b.Program} + args = append(args, b.ObjectFiles...) + args = append(args, b.Oflags...) + args = append(args, "-L", fromRoot("/arch/$ARCH/lib")) + args = append(args, b.Libs...) + run(b, *shellhack, exec.Command(tools["ld"], args...)) +} + +func install(b *build) { + if b.Install == "" { + return + } + + log.Printf("Installing %s\n", b.name) + failOn(os.MkdirAll(b.Install, 0755)) + + switch { + case len(b.SourceFilesCmd) > 0: + for _, n := range b.SourceFilesCmd { + ext := filepath.Ext(n) + exe := n[:len(n)-len(ext)] + move(exe, b.Install) + } + case len(b.Program) > 0: + move(b.Program, b.Install) + case len(b.Library) > 0: + libpath := path.Join(b.Install, b.Library) + args := append([]string{"-rs", libpath}, b.ObjectFiles...) + run(b, *shellhack, exec.Command(tools["ar"], args...)) + run(b, *shellhack, exec.Command(tools["ranlib"], libpath)) + } +} + +func move(from, to string) { + final := path.Join(to, from) + log.Printf("move %s %s\n", from, final) + _ = os.Remove(final) + failOn(os.Link(from, final)) + failOn(os.Remove(from)) +} + +func run(b *build, pipe bool, cmd *exec.Cmd) { + if b != nil { + cmd.Env = append(os.Environ(), b.Env...) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if pipe { + // Sh sends cmd to a shell. It's needed to enable $LD_PRELOAD tricks, see https://github.com/Harvey-OS/jehanne/issues/8#issuecomment-131235178 + shell := exec.Command(tools["sh"]) + shell.Env = cmd.Env + shell.Stderr = os.Stderr + shell.Stdout = os.Stdout + + commandString := cmd.Args[0] + commandString += " " + strings.Join(wrapInQuote(cmd.Args[1:]), " ") + shStdin, err := shell.StdinPipe() + if err != nil { + log.Fatalf("cannot pipe [%v] to %s: %v", commandString, tools["sh"], err) + } + go func() { + defer shStdin.Close() + io.WriteString(shStdin, commandString) + }() + + log.Printf("%q | %s\n", commandString, tools["sh"]) + failOn(shell.Run()) + return + } + log.Println(strings.Join(cmd.Args, " ")) + failOn(cmd.Run()) +} + +func projects(b *build, r []*regexp.Regexp) { + for _, v := range b.Projects { + f, _ := findBuildfile(v) + log.Printf("Doing %s\n", f) + project(f, r) + } +} + +// assumes we are in the wd of the project. +func project(bf string, which []*regexp.Regexp) { + cwd, err := os.Getwd() + failOn(err) + debug("Start new project cwd is %v", cwd) + defer os.Chdir(cwd) + dir := path.Dir(bf) + root := path.Base(bf) + debug("CD to %v and build using %v", dir, root) + failOn(os.Chdir(dir)) + builds := process(root, which) + debug("Processing %v: %d target", root, len(builds)) + for _, b := range builds { + debug("Processing %v: %v", b.name, b) + projects(&b, regexpAll) + for _, c := range b.Pre { + // this is a hack: we just pass the command through as an exec.Cmd + run(&b, true, exec.Command(c)) + } + buildkernel(&b) + if len(b.SourceFiles) > 0 || len(b.SourceFilesCmd) > 0 { + compile(&b) + } + if b.Program != "" || len(b.SourceFilesCmd) > 0 { + link(&b) + } + install(&b) + for _, c := range b.Post { + run(&b, true, exec.Command(c)) + } + } +} + +func main() { + // A small amount of setup is done in the paths*.go files. They are + // OS-specific path setup/manipulation. "jehanne" is set there and $PATH is + // adjusted. + var err error + findTools(os.Getenv("TOOLPREFIX")) + flag.Parse() + cwd, err = os.Getwd() + failOn(err) + + a := os.Getenv("ARCH") + if a == "" || !arch[a] { + s := []string{} + for i := range arch { + s = append(s, i) + } + log.Fatalf("You need to set the ARCH environment variable from: %v", s) + } + + // ensure this is exported, in case we used a default value + os.Setenv("JEHANNE", jehanne) + + if os.Getenv("LD_PRELOAD") != "" { + log.Println("Using shellhack") + *shellhack = true + } + + // If no args, assume 'build.json' + // Otherwise the first argument is either + // - the path to a json file + // - a directory containing a 'build.json' file + // - a regular expression to apply assuming 'build.json' + // Further arguments are regular expressions. + consumedArgs := 0; + bf := "" + if len(flag.Args()) == 0 { + f, err := findBuildfile("build.json") + failOn(err) + bf = f + } else { + f, err := findBuildfile(flag.Arg(0)) + failOn(err) + + if f == "" { + f, err := findBuildfile("build.json") + failOn(err) + bf = f + } else { + consumedArgs = 1 + bf = f + } + } + + re := []*regexp.Regexp{regexp.MustCompile(".")} + if len(flag.Args()) > consumedArgs { + re = re[:0] + for _, r := range flag.Args()[consumedArgs:] { + rx, err := regexp.Compile(r) + failOn(err) + re = append(re, rx) + } + } + project(bf, re) +} + +func findTools(toolprefix string) { + var err error + for k, v := range tools { + if x := os.Getenv(strings.ToUpper(k)); x != "" { + v = x + } + v, err = exec.LookPath(toolprefix + v) + failOn(err) + tools[k] = v + } +} + +// disambiguate the buildfile argument +func findBuildfile(f string) (string, error) { + if strings.HasSuffix(f, ".json"){ + if fi, err := os.Stat(f); err == nil && !fi.IsDir() { + return f, nil + } + return "", fmt.Errorf("unable to find buildfile %s", f) + } + if strings.Contains(f, "/") { + return findBuildfile(path.Join(f, "build.json")) + } + return "", nil +} diff --git a/src/jehanne/cmd/build/codegen.go b/src/jehanne/cmd/build/codegen.go new file mode 100644 index 0000000..74a2403 --- /dev/null +++ b/src/jehanne/cmd/build/codegen.go @@ -0,0 +1,196 @@ +package main + +import ( + "bytes" + "debug/elf" + "fmt" + "io/ioutil" + "os" + "os/exec" + "text/template" +) + +const kernconfTmpl = ` +#include "u.h" +#include "../port/lib.h" +#include "mem.h" +#include "dat.h" +#include "fns.h" +#include "../port/error.h" +#include "io.h" + +void +rdb(void) +{ + splhi(); + iprint("rdb...not installed\n"); + for(;;); +} + +{{ range .Rootcodes }} +{{ . }} +{{ end }} + +{{ range .Config.Dev }}extern Dev {{ . }}devtab; +{{ end }} +Dev *devtab[] = { +{{ range .Config.Dev }} + &{{ . }}devtab, +{{ end }} + nil, +}; + +{{ range .Config.Link }}extern void {{ . }}link(void); +{{ end }} +void +links(void) +{ +{{ range .Rootnames }}addbootfile("{{ . }}", ramfs_{{ . }}_code, ramfs_{{ . }}_len); +{{ end }} +{{ range .Config.Link }}{{ . }}link(); +{{ end }} +} + +#include "../ip/ip.h" +{{ range .Config.Ip }}extern void {{ . }}init(Fs*); +{{ end }} +void (*ipprotoinit[])(Fs*) = { +{{ range .Config.Ip }} {{ . }}init, +{{ end }} + nil, +}; + +#include "../port/sd.h" +{{ range .Config.Sd }}extern SDifc {{ . }}ifc; +{{ end }} +SDifc* sdifc[] = { +{{ range .Config.Sd }} &{{ . }}ifc, +{{ end }} + nil, +}; + +{{ range .Config.Uart }}extern PhysUart {{ . }}physuart; +{{ end }} +PhysUart* physuart[] = { +{{ range .Config.Uart }} &{{ . }}physuart, +{{ end }} + nil, +}; + +#define Image IMAGE +#include +#include +#include +#include "screen.h" +{{ range .Config.VGA }}extern VGAdev {{ . }}dev; +{{ end }} +VGAdev* vgadev[] = { +{{ range .Config.VGA }} &{{ . }}dev, +{{ end }} + nil, +}; + +{{ range .Config.VGA }}extern VGAcur {{ . }}cur; +{{ end }} +VGAcur* vgacur[] = { +{{ range .Config.VGA }} &{{ . }}cur, +{{ end }} + nil, +}; + +Physseg physseg[8] = { + { + .attr = SG_SHARED, + .name = "shared", + .size = SEGMAXPG, + }, + { + .attr = SG_BSS, + .name = "memory", + .size = SEGMAXPG, + }, +}; +int nphysseg = 8; + +{{ range .Config.Code }}{{ . }} +{{ end }} + +char* conffile = "{{ .Path }}"; + +` + +// These are the two big code generation functions. + +// data2c takes the file at path and creates a C byte array containing it. +func data2c(name string, path string) string { + var out []byte + var in []byte + + if elf, err := elf.Open(path); err == nil { + elf.Close() + cwd, err := os.Getwd() + tmpf, err := ioutil.TempFile(cwd, name) + failOn(err) + + run(nil, *shellhack, exec.Command(tools["strip"], "-o", tmpf.Name(), path)) + + in, err = ioutil.ReadAll(tmpf) + failOn(err) + + tmpf.Close() + os.Remove(tmpf.Name()) + } else { + var file *os.File + var err error + + file, err = os.Open(path) + failOn(err) + + in, err = ioutil.ReadAll(file) + failOn(err) + + file.Close() + } + + total := len(in) + + out = []byte(fmt.Sprintf("static unsigned char ramfs_%s_code[] = {\n", name)) + for len(in) > 0 { + for j := 0; j < 16 && len(in) > 0; j++ { + out = append(out, []byte(fmt.Sprintf("0x%02x, ", in[0]))...) + in = in[1:] + } + out = append(out, '\n') + } + + out = append(out, []byte(fmt.Sprintf("0,\n};\nint ramfs_%s_len = %v;\n", name, total))...) + + return string(out) +} + +// confcode creates a kernel configuration header. +func confcode(path string, kern *kernel) []byte { + var rootcodes []string + var rootnames []string + for name, path := range kern.Ramfiles { + code := data2c(name, fromRoot(path)) + rootcodes = append(rootcodes, code) + rootnames = append(rootnames, name) + } + + vars := struct { + Path string + Config kernconfig + Rootnames []string + Rootcodes []string + }{ + path, + kern.Config, + rootnames, + rootcodes, + } + tmpl := template.Must(template.New("kernconf").Parse(kernconfTmpl)) + codebuf := &bytes.Buffer{} + failOn(tmpl.Execute(codebuf, vars)) + return codebuf.Bytes() +} diff --git a/src/jehanne/cmd/build/doc.go b/src/jehanne/cmd/build/doc.go new file mode 100644 index 0000000..58bf91f --- /dev/null +++ b/src/jehanne/cmd/build/doc.go @@ -0,0 +1,184 @@ +/* +BUILDFILE FORMAT + +A buildfile is a json object containing build objects. By convention, it's +broken out onto multiple lines indented by tabs, and the keys are TitleCased. + +The environment varibles JEHANNE and ARCH are guarenteed to be set for the +buildfile. PATH is modified so that "$JEHANNE/hacking" is included. + +Any key that takes a path or array of paths as its value has absolute paths +re-rooted to the jehanne tree and variables in the string expanded once. + +The shell commands in the "Pre" and "Post" steps must be in the subset of +syntax that's accepted by both POSIX sh and rc. Practically, this means +arguments with a "=" must be single-quoted, "test" must be called as +"test" (not "["), "if" statements may not have an "else" clause, "switch" +statements may not be used, "for" statements may only have one body command, +and redirection to a file descriptor cannont be used. + +BUILD OBJECT + +A build object has the following keys and types: + + Projects []string + Pre []string + Post []string + Cflags []string + Oflags []string + Include []string + SourceFiles []string + ObjectFiles []string + Libs []string + Env []string + SourceFilesCmd []string + Program string + Library string + Install string + Kernel kernel + +These are the steps taken, in order: + + Env + Include + Projects + Pre + Kernel + [compile] SourceFiles SourceFilesCmd Cflags Program + [link] ObjectFiles Libs Oflags Library + Install + Post + +"[compile]" and "[link]" are steps synthesized from the specified keys. + +The meaning of the keys is as follows: + +"Env" is an array of environment variables to be put in the environment of +every command run in a build. They are expanded once and only available for +use in other steps. + +"Include" is an array of buildfiles to "merge" into the current one. This +is done before other projects are built. + +"Projects" is an array of buildfiles to build in their entirety before +starting the current build. + +"Pre" is an array of commands to run before starting the build. + +"Kernel" is a kernel build object. See the kernel object section. + +"Cflags" is an array of flags to pass to the C compiler. They are in addition +to the standard flags of + + -std=c11 -c -I /$ARCH/include -I /sys/include -I . + +The standard include paths are re-rooted to the jehanne tree if not on a jehanne +system. + +"SourceFilesCmd" is an array of C files where each one should result in an +executable. If this key is provided, "SourceFiles" and "Program" are ignored. + +"SourceFiles" is an array of C files that will ultimately produce a single +binary or library, named by "Program" or "Library" respectively. + +"Program" is the name of the final executable for the files specified by +"SourceFiles". + +"Oflags" is an array of flags to pass to the linker. They are in addition +to the standard flags of + + -o $program $objfiles -L /$ARCH/lib $libs + +The lib path is re-rooted to the jehanne tree if not on a jehanne system. + +"ObjectFiles" is an array of strings specifying object files to be linked +into the final program specified by "Program". Any object files produced +by the preceeding "[compile]" step are automatically added to this before +beginning the "[link]" step. + +"Libs" is a an array of library arguments to pass to the linker. + +"Library" is the name of the archive resulting from bundling "SourceFiles" +into a library. The resulting archive has 'ranlib' run on it automatically. + +"Install" is a directory where the result of the "[link]" step is moved +to. If it does not exist, it is created. + +"Post" is an array of commands to run last during the build. + +KERNEL OBJECT + +A build object has the following keys and types: + + Systab string + Ramfiles map[string]string + Config { + Code []string + Dev []string + Ip []string + Link []string + Sd []string + Uart []string + VGA []string + } + +"Systab" is the header that defines the syscall table. + +"Ramfiles" is an object of name, path pairs of binaries at "path" that will +be baked into the kernel and available at a binary named "name". + +"Config" is an object used to generate the kernel configuration source. "Dev", +"Ip", "Sd", "Uart", "Link", and "VGA" control which drivers of the various +types are included. "Code" is lines of arbitrary C code. + +*/ +package main + +import ( + "flag" + "fmt" + "os" + "strings" +) + +var helptext = ` +The buildfile is looked for at these positions, in this order: + + ./$arg + ./$arg/build.json + /sys/src/$arg.json + /sys/src/$arg/build.json + +If the buildfile argument is not provided, it defaults to "build.json". + +After the buildfile, a number of regexps specifying targets may be provided. +If a target matches any supplied regexp, it is acted on. These regexps only +apply to the top-level buildfile. + +BUILDFILE + +See the build godoc for more information about the buildfile format. + +ENVIRONMENT + +ARCH is needed. Current acceptable vaules are: amd64 + +JEHANNE may be supplied to point at a jehanne tree. +The default on Jehanne is "/". +The default on Linux and OSX is to attempt to find the top level of a git +repository. +` + +func init() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] [buildfile] [target...]\n\n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprintln(os.Stderr, helptext) + fmt.Fprintln(os.Stderr, "Tools to be used with current settings:") + fmt.Fprintf(os.Stderr, " prefix ($TOOLPREFIX): %q\n", os.Getenv("TOOLPREFIX")) + for k, v := range tools { + fmt.Fprintf(os.Stderr, " %s ($%s): %s\n", k, strings.ToUpper(k), v) + } + } +} diff --git a/src/jehanne/cmd/build/paths.go b/src/jehanne/cmd/build/paths.go new file mode 100644 index 0000000..083b1f1 --- /dev/null +++ b/src/jehanne/cmd/build/paths.go @@ -0,0 +1,31 @@ +// +build !jehanne + +package main + +import ( + "log" + "os" + "os/exec" + "strings" +) + +func init() { + jehanne = os.Getenv("JEHANNE") + if jehanne != "" { + return + } + // git is purely optional, for lazy people. + out, err := exec.Command("git", "rev-parse", "--show-toplevel").Output() + if err == nil { + jehanne = strings.TrimSpace(string(out)) + hackingAt := strings.LastIndex("/hacking", jehanne) + if(hackingAt >= 0){ + jehanne = jehanne[0:hackingAt] + } + } + if jehanne == "" { + log.Fatal("Set the JEHANNE environment variable or run from a git checkout.") + } + + os.Setenv("PATH", strings.Join([]string{fromRoot("/hacking"), os.Getenv("PATH")}, string(os.PathListSeparator))) +} diff --git a/src/jehanne/cmd/build/paths_harvey.go b/src/jehanne/cmd/build/paths_harvey.go new file mode 100644 index 0000000..c293c4b --- /dev/null +++ b/src/jehanne/cmd/build/paths_harvey.go @@ -0,0 +1,17 @@ +// +build jehanne + +package main + +import ( + "os" + "strings" +) + +func init() { + jehanne = os.Getenv("JEHANNE") + if jehanne != "" { + return + } + jehanne = "/" + os.Setenv("path", strings.Join([]string{fromRoot("/util"), os.Getenv("path")}, string(os.PathListSeparator))) +} diff --git a/src/jehanne/cmd/convert/convert.go b/src/jehanne/cmd/convert/convert.go new file mode 100644 index 0000000..1632292 --- /dev/null +++ b/src/jehanne/cmd/convert/convert.go @@ -0,0 +1,155 @@ +package main + +import ( + "encoding/json" + "flag" + "io/ioutil" + "log" + "os" + "path/filepath" +) + +var config = struct { + Jehanne string + Args []string + Except map[string]bool + CmdName string + FullPath string + Src string + Uroot string + Cwd string + Bbsh string + + Goroot string + Gosrcroot string + Arch string + Goos string + Gopath string + TempDir string + Go string + Debug bool + Fail bool +}{ + Jehanne: os.Getenv("JEHANNE"), + Except: map[string]bool{"sysconf.json": true}, +} + +type fixup func(string, map[string]interface{}) + +func cflags(n string, jsmap map[string]interface{}) { + if _, ok := jsmap["Cflags"]; ok { + log.Printf("Deleting Cflags from %v", n) + delete(jsmap, "Cflags") + // TODO: once we have another ARCH, use it. + a := []string{"/arch/amd64/include/cflags.json"} + switch tval := jsmap["Include"].(type) { + case []interface{}: + for _, v := range tval { + a = append(a, v.(string)) + } + } + jsmap["Include"] = a + } +} + +func removeempty(n string, jsmap map[string]interface{}) { + for key, val := range jsmap { + switch tval := val.(type) { + case map[string]interface{}: + log.Printf("%s: tval %s", n, tval) + if len(tval) == 0 { + delete(jsmap, key) + } + case []interface{}: + if len(tval) == 0 { + delete(jsmap, key) + } + } + } +} + +func checkname(n string, jsmap map[string]interface{}) { + if _, ok := jsmap["Name"]; !ok { + log.Print("File %v has no \"Name\" key", n) + } +} + +func one(n string, f ...fixup) error { + buf, err := ioutil.ReadFile(n) + if err != nil { + return err + } + + var jsmap map[string]interface{} + if err := json.Unmarshal(buf, &jsmap); err != nil { + return err + } + if config.Debug { + log.Printf("%v: %v", n, jsmap) + } + + mapname := jsmap["Name"].(string) + delete(jsmap, "Name") + var nmap = make(map[string]map[string]interface{}) + nmap[mapname] = jsmap + buf, err = json.MarshalIndent(nmap, "", "\t") + if err != nil { + return err + } + buf = append(buf, '\n') + + if config.Debug { + os.Stdout.Write(buf) + } else { + ioutil.WriteFile(n, buf, 0666) + } + return nil +} + +func init() { + if config.Jehanne == "" { + log.Fatalf("Please set $JEHANNE") + } + + if len(config.Args) == 0 { + config.Args = []string{config.Jehanne} + } +} + +func main() { + var err error + + flag.BoolVar(&config.Debug, "d", true, "Enable debug prints") + flag.Parse() + config.Args = flag.Args() + for _, n := range config.Args { + err = filepath.Walk(n, func(name string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if fi.IsDir() { + return nil + } + n := fi.Name() + if len(n) < 5 || n[len(n)-5:] != ".json" { + return nil + } + if config.Except[fi.Name()] { + return nil + } + todo := []fixup{removeempty, checkname} + log.Printf("process %s", name) + err = one(name, todo...) + if err != nil { + log.Printf("%s: %s\n", name, err) + return err + } + return nil + }) + + if err != nil { + log.Fatal("%v", err) + + } + } +} diff --git a/src/jehanne/cmd/data2c/data2c.go b/src/jehanne/cmd/data2c/data2c.go new file mode 100644 index 0000000..ab3b5d5 --- /dev/null +++ b/src/jehanne/cmd/data2c/data2c.go @@ -0,0 +1,39 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" +) + +func main() { + flag.Parse() + a := flag.Args() + if len(a) != 2 { + fmt.Fprintf(os.Stderr, "[%v]usage: data2s name input-file (writes to stdout)\n", a) + os.Exit(1) + } + + n := a[0] + i := a[1] + in, err := ioutil.ReadFile(i) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + + total := len(in) + + fmt.Printf("unsigned char %vcode[] = {\n", n) + for len(in) > 0 { + for j := 0; j < 16 && len(in) > 0; j++ { + fmt.Printf("0x%02x, ", in[0]) + in = in[1:] + } + fmt.Printf("\n") + + } + + fmt.Printf("0,\n};\nint %vlen = %v;\n", n, total) +} diff --git a/src/jehanne/cmd/elf2c/elf2c.go b/src/jehanne/cmd/elf2c/elf2c.go new file mode 100644 index 0000000..6cd8e06 --- /dev/null +++ b/src/jehanne/cmd/elf2c/elf2c.go @@ -0,0 +1,113 @@ +package main + +import ( + "bufio" + "debug/elf" + "flag" + "fmt" + "io" + "math" + "os" + "path" +) + +var dry = flag.Bool("dryrun", true, "don't really do it") + +func gencode(w io.Writer, n, t string, m []byte, start, end uint64) { + fmt.Fprintf(os.Stderr, "Write %v %v start %v end %v\n", n, t, start, end) + fmt.Fprintf(w, "int %v_%v_start = %v;\n", n, t, start) + fmt.Fprintf(w, "int %v_%v_end = %v;\n", n, t, end) + fmt.Fprintf(w, "int %v_%v_len = %v;\n", n, t, end-start) + fmt.Fprintf(w, "uint8_t %v_%v_out[] = {\n", n, t) + for i := uint64(start); i < end; i += 16 { + for j := uint64(0); i+j < end && j < 16; j++ { + fmt.Fprintf(w, "0x%02x, ", m[j+i]) + } + fmt.Fprintf(w, "\n") + } + fmt.Fprintf(w, "};\n") +} +func main() { + flag.Parse() + a := flag.Args() + for _, n := range a { + f, err := elf.Open(n) + if err != nil { + fmt.Printf("%v %v\n", n, err) + continue + } + var dataend, codeend, end uint64 + var datastart, codestart, start uint64 + datastart, codestart, start = math.MaxUint64, math.MaxUint64, math.MaxUint64 + mem := []byte{} + for _, v := range f.Progs { + if v.Type != elf.PT_LOAD { + continue + } + fmt.Fprintf(os.Stderr, "processing %v\n", v) + // MUST alignt to 2M page boundary. + // then MUST allocate a []byte that + // is the right size. And MUST + // see if by some off chance it + // joins to a pre-existing segment. + // It's easier than it seems. We produce ONE text + // array and ONE data array. So it's a matter of creating + // a virtual memory space with an assumed starting point of + // 0x200000, and filling it. We just grow that as needed. + + curstart := v.Vaddr & ^uint64(0xfff) // 0x1fffff) + curend := v.Vaddr + v.Memsz + fmt.Fprintf(os.Stderr, "s %x e %x\n", curstart, curend) + if curend > end { + nmem := make([]byte, curend) + copy(nmem, mem) + mem = nmem + } + if curstart < start { + start = curstart + } + + if v.Flags&elf.PF_X == elf.PF_X { + if curstart < codestart { + codestart = curstart + } + if curend > codeend { + codeend = curend + } + fmt.Fprintf(os.Stderr, "code s %v e %v\n", codestart, codeend) + } else { + if curstart < datastart { + datastart = curstart + } + if curend > dataend { + dataend = curend + } + fmt.Fprintf(os.Stderr, "data s %v e %v\n", datastart, dataend) + } + for i := uint64(0); i < v.Filesz; i++ { + if amt, err := v.ReadAt(mem[v.Vaddr+i:], int64(i)); err != nil && err != io.EOF { + fmt.Fprintf(os.Stderr, "%v: %v\n", amt, err) + os.Exit(1) + } else if amt == 0 { + if i < v.Filesz { + fmt.Fprintf(os.Stderr, "%v: Short read: %v of %v\n", v, i, v.Filesz) + os.Exit(1) + } + break + } else { + i = i + uint64(amt) + fmt.Fprintf(os.Stderr, "i now %v\n", i) + } + } + fmt.Fprintf(os.Stderr, "Processed %v\n", v) + } + fmt.Fprintf(os.Stderr, "gencode\n") + // Gen code to stdout. For each file, create an array, a start, and an end variable. + w := bufio.NewWriter(os.Stdout) + _, file := path.Split(n) + gencode(w, file, "code", mem, codestart, codeend) + gencode(w, file, "data", mem, datastart, dataend) + w.Flush() + } + +} diff --git a/src/jehanne/cmd/jsonpretty/jsonpretty.go b/src/jehanne/cmd/jsonpretty/jsonpretty.go new file mode 100644 index 0000000..7938ba2 --- /dev/null +++ b/src/jehanne/cmd/jsonpretty/jsonpretty.go @@ -0,0 +1,53 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" +) + +func main() { + if len(os.Args) < 2 { + fmt.Printf("usage: jsonpretty input.json [output.json]\n") + os.Exit(1) + } + + buf, err := ioutil.ReadFile(os.Args[1]) + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } + + var jsmap map[string]interface{} + if err := json.Unmarshal(buf, &jsmap); err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } + + for key, val := range jsmap { + switch tval := val.(type) { + case map[string]interface{}: + if len(tval) == 0 { + delete(jsmap, key) + } + case []interface{}: + if len(tval) == 0 { + delete(jsmap, key) + } + } + } + + buf, err = json.MarshalIndent(jsmap, "", "\t") + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } + buf = append(buf, '\n') + + if len(os.Args) == 3 { + ioutil.WriteFile(os.Args[2], buf, 0666) + } else { + os.Stdout.Write(buf) + } +} diff --git a/src/jehanne/cmd/kpdump/kpdump.go b/src/jehanne/cmd/kpdump/kpdump.go new file mode 100644 index 0000000..74bbf16 --- /dev/null +++ b/src/jehanne/cmd/kpdump/kpdump.go @@ -0,0 +1,115 @@ +package main + +import ( + "debug/elf" + "encoding/binary" + "flag" + "fmt" + "io" + "math" + "os" +) + +const LRES = 3 + +var kernel = flag.String("k", "9k", "kernel name") + +func main() { + flag.Parse() + + n := flag.Args()[0] + d, err := os.Open(n) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + f, err := elf.Open(*kernel) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + + var codeend uint64 + var codestart uint64 = math.MaxUint64 + + for _, v := range f.Progs { + if v.Type != elf.PT_LOAD { + continue + } + fmt.Fprintf(os.Stderr, "processing %v\n", v) + // MUST alignt to 2M page boundary. + // then MUST allocate a []byte that + // is the right size. And MUST + // see if by some off chance it + // joins to a pre-existing segment. + // It's easier than it seems. We produce ONE text + // array and ONE data array. So it's a matter of creating + // a virtual memory space with an assumed starting point of + // 0x200000, and filling it. We just grow that as needed. + + curstart := v.Vaddr + curend := v.Vaddr + v.Memsz + // magic numbers, BAH! + if curstart < uint64(0xffffffff00000000) { + curstart += 0xfffffffff0000000 + curend += 0xfffffffff0000000 + } + fmt.Fprintf(os.Stderr, "s %x e %x\n", curstart, curend) + if v.Flags&elf.PF_X == elf.PF_X { + if curstart < codestart { + codestart = curstart + } + if curend > codeend { + codeend = curend + } + fmt.Fprintf(os.Stderr, "code s %x e %x\n", codestart, codeend) + } + } + fmt.Fprintf(os.Stderr, "code s %x e %x\n", codestart, codeend) + s, err := f.Symbols() + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + // maybe we should stop doing LRES ... + symname := make([]string, codeend-codestart) + for i := range symname { + symname[i] = fmt.Sprintf("[0x%x]", codestart+uint64(i)) + } + for _, v := range s { + vstart := v.Value + vend := v.Value + v.Size + if v.Value > codeend { + continue + } + if v.Value+v.Size < codestart { + continue + } + if vstart < codestart { + v.Value = codestart + } + if vend > codeend { + vend = codeend + } + for i := vstart; i < vend; i++ { + symname[i-codestart] = v.Name + } + } + symname[0] = "Total ms" + symname[1< 0 { + fmt.Printf("%s %d\n", symname[pc-codestart], count) + } + pc += (1 << LRES) + } +} diff --git a/src/jehanne/cmd/mksys/mksys.go b/src/jehanne/cmd/mksys/mksys.go new file mode 100644 index 0000000..2595cec --- /dev/null +++ b/src/jehanne/cmd/mksys/mksys.go @@ -0,0 +1,325 @@ +/* + * This file is part of the UCB release of Plan 9. It is subject to the license + * terms in the LICENSE file found in the top-level directory of this + * distribution and at http://akaros.cs.berkeley.edu/files/Plan9License. No + * part of the UCB release of Plan 9, including this file, may be copied, + * modified, propagated, or distributed except according to the terms contained + * in the LICENSE file. + */ + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "strings" + "text/template" +) + +type Syscall struct { + Ret []string + Args []string + Name string + Id uint32 + Define string + Sysname string + Libname string + Fudge string `json:"-"` + GoArgs []string `json:"-"` + Ret0 string `json:"-"` +} + +type Syserror struct { + Name string + String string + Id uint32 +} + +type Bootmethods struct { + Name string + Config string + Connect string + Arg string +} + +type Sysconf struct { + Syscalls []Syscall + Syserrors []Syserror + Bootmethods []Bootmethods +} + +var mode = flag.String("mode", "", "must be one of: sys.h, sysdecl.h, syscallfiles, systab.c, error.h, errstr.h, sys_jehanne.s, sysnum.go") +var outpath = flag.String("o", "", "path/to/output.c") + +func usage(msg string) { + fmt.Fprint(os.Stderr, msg) + fmt.Fprint(os.Stderr, "Usage: mksys [-o outpath] -mode=MODE path/to/sysconf.json\n") + flag.PrintDefaults() + os.Exit(1) +} + +func main() { + + flag.Parse() + + if flag.NArg() != 1 { + usage("no path to sysconf.json") + } + + outfile := os.Stdout + if *mode != "syscallfiles" && *outpath != "" { + of, err := os.Create(*outpath) + if err != nil { + log.Fatal(err) + } + outfile = of + } + + buf, err := ioutil.ReadFile(flag.Arg(0)) + if err != nil { + log.Fatal(err) + } + + var sysconf Sysconf + err = json.Unmarshal(buf, &sysconf) + if err != nil { + log.Fatal(err) + } + + syscalls := sysconf.Syscalls + syserrors := sysconf.Syserrors + bootmethods := sysconf.Bootmethods + for i := range syscalls { + if syscalls[i].Define == "" { + syscalls[i].Define = strings.ToUpper(syscalls[i].Name) + } + if syscalls[i].Sysname == "" { + syscalls[i].Sysname = "sys" + syscalls[i].Name + } + if syscalls[i].Libname == "" { + syscalls[i].Libname = syscalls[i].Name + } + } + + switch *mode { + case "sys_jehanne.s": + if os.Getenv("ARCH") != "amd64" { + usage("ARCH unsupported or not set") + } + syscallargs := []string{"DI", "SI", "DX", "R10", "R8", "R9"} + //funcallregs := []string{ "DI", "SI", "DX", "CX", "R8", "R9" }; + for i := range syscalls { + goargs := []string{} + fpoff := 0 + for k := range syscalls[i].Args { + switch syscalls[i].Args[k] { + case "int32_t", "uint32_t": + goargs = append(goargs, fmt.Sprintf("MOVL arg%d+%d(FP), %s", k, fpoff, syscallargs[k])) + fpoff += 4 + case "void*", "char*", "char**", "uint8_t*", "int32_t*", "uint64_t*", "int64_t*", "int64_t": + fpoff = (fpoff + 7) & ^7 + goargs = append(goargs, fmt.Sprintf("MOVQ arg%d+%d(FP), %s", k, fpoff, syscallargs[k])) + fpoff += 8 + default: + log.Fatalf("unsupported arg %s in syscall: %v", syscalls[i].Args[k], syscalls[i]) + } + } + syscalls[i].GoArgs = goargs + switch syscalls[i].Ret[0] { + case "int32_t", "uint32_t": + syscalls[i].Ret0 = fmt.Sprintf("MOVL AX, ret+%d(FP)", fpoff) + fpoff += 4 + case "void*", "char*", "char**", "uint8_t*", "int32_t*", "uint64_t*", "int64_t*", "int64_t": + fpoff = (fpoff + 7) & ^7 + syscalls[i].Ret0 = fmt.Sprintf("MOVQ AX, ret+%d(FP)", fpoff) + fpoff += 8 + default: + log.Fatalf("unsupported Ret[0] in syscall: %v", syscalls[i]) + } + } + tmpl, err := template.New("sys_jehanne.s").Parse(`/* automatically generated by mksys */ +/* System calls for AMD64, Jehanne */ +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" +{{ range . }} +TEXT runtime·{{ .Libname }}(SB),NOSPLIT,$0 +{{ range .GoArgs }} {{ . }} +{{ end }} MOVQ ${{ .Id }}, AX + SYSCALL + {{ .Ret0 }} + RET +{{ end }} +`) + if err != nil { + log.Fatal(err) + } + + err = tmpl.Execute(outfile, syscalls) + if err != nil { + log.Fatal(err) + } + + case "syscallfiles": + if os.Getenv("ARCH") != "amd64" { + usage("ARCH unsupported or not set") + } + tmpl, err := template.New("syscall.s").Parse(`/* automatically generated by mksys */ +.globl {{ .Libname }} +{{ .Libname }}: + movq %rcx, %r10 /* rcx gets smashed by systenter. Use r10.*/ + movq ${{ .Id }},%rax /* Put the system call into rax, just like linux. */ + syscall + ret +`) + if err != nil { + log.Fatal(err) + } + + for i := range syscalls { + + path := path.Join(*outpath, syscalls[i].Libname+".s") + file, err := os.Create(path) + if err != nil { + log.Fatal(err) + } + + err = tmpl.Execute(file, syscalls[i]) + if err != nil { + log.Fatal(err) + } + + err = file.Close() + if err != nil { + log.Fatal(err) + } + } + case "sysnum.go": + tmpl, err := template.New("sysnum.go").Parse(`// automatically generated by mksys +package syscall + +const( +{{ range . }} SYS_{{ .Define }} = {{ .Id }} +{{ end }} +) +`) + err = tmpl.Execute(outfile, syscalls) + if err != nil { + log.Fatal(err) + } + + case "sys.h": + tmpl, err := template.New("sys.h").Parse(`/* automatically generated by mksys */ +{{ range . }}#define {{ .Define }} {{ .Id }} +{{ end }} +`) + err = tmpl.Execute(outfile, syscalls) + if err != nil { + log.Fatal(err) + } + + case "sysdecl.h": + tmpl, err := template.New("sysdecl.h").Parse(`/* automatically generated by mksys */ +{{ range . }}extern {{ .Ret0 }} {{ .Libname }}({{ range $i, $e := .Args }}{{ if $i }}, {{ end }}{{ $e }}{{ end }}); +{{ end }} +`) + err = tmpl.Execute(outfile, syscalls) + if err != nil { + log.Fatal(err) + } + + case "systab.c": + for i := range syscalls { + var fudge string + switch syscalls[i].Ret[0] { + case "int32_t": + fudge = "{ .i = -1 }" + case "int64_t": + fudge = "{ .vl = -1ll }" + case "void*", "char*": + fudge = "{ .v = (void*)-1ll }" + default: + log.Fatalf("unsupported Ret[0] in syscall: %v", syscalls[i]) + } + if syscalls[i].Fudge == "" { + syscalls[i].Fudge = fudge + } + + syscalls[i].Ret0 = syscalls[i].Ret[0] + } + tmpl, err := template.New("systab.c").Parse(`/* automatically generated by mksys */ +#include "u.h" +#include "../port/lib.h" +#include "mem.h" +#include "dat.h" +#include "fns.h" +#include "../../libc/9syscall/sys.h" + +{{ range . }}extern void {{ .Sysname }}(Ar0*, ...); +{{ end }} +Systab systab[] = { +{{ range . }}[{{ .Define }}] { "{{ .Name }}", {{ .Sysname }}, {{ .Fudge }} }, +{{ end }} +}; +int nsyscall = nelem(systab); +`) + err = tmpl.Execute(outfile, syscalls) + if err != nil { + log.Fatal(err) + } + + case "error.h": + tmpl, err := template.New("error.h").Parse(`/* automatically generated by mksys */ +{{ range . }}extern char {{ .Name }}[]; /* {{ .String }} */ +{{ end }} +`) + err = tmpl.Execute(outfile, syserrors) + if err != nil { + log.Fatal(err) + } + + case "errstr.h": + tmpl, err := template.New("errstr.h").Parse(`/* automatically generated by mksys */ +{{ range . }}char {{ .Name }}[] = "{{ .String }}"; +{{ end }} +`) + err = tmpl.Execute(outfile, syserrors) + if err != nil { + log.Fatal(err) + } + case "bootamd64cpu.c": + tmpl, err := template.New("bootamd64cpu.c").Parse(`/* automatically generated by mksys */ +#include +#include + +#include "../boot/boot.h" + +Method method[] = { +{{ range . }}{ "{{.Name}}", {{.Config}}, {{.Connect}}, "{{.Arg}}", }, +{{ end }} + { nil }, +}; + +int cpuflag = 1; +char* rootdir = "/root"; +char* bootdisk = "#S/sdE0/"; +extern void boot(int, char**); + +void +main(int argc, char **argv) +{ + boot(argc, argv); +} +int (*cfs)(int) = 0; +`) + err = tmpl.Execute(outfile, bootmethods) + if err != nil { + log.Fatal(err) + } + } +} diff --git a/src/jehanne/cmd/op2/op2.go b/src/jehanne/cmd/op2/op2.go new file mode 100644 index 0000000..045d87f --- /dev/null +++ b/src/jehanne/cmd/op2/op2.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/binary" + "fmt" + "io" + "os" +) + +/* +first word +high 8 bits is ee, which is an invalid address on amd64. +next 8 bits is protocol version +next 16 bits is unused, MBZ. Later, we can make it a packet type. +next 16 bits is core id +next 8 bits is unused +next 8 bits is # words following. + +second word is time in ns. (soon to be tsc ticks) + +Third and following words are PCs, there must be at least one of them. +*/ +type sample struct { + Wordcount, _ uint8 + Coreid uint16 + _ uint16 + Version, Marker uint8 + Ns uint64 +} + +type backtrace struct { + Pcs []uint64 +} + +/* the docs lie. Perl expects Count to be zero. I only wasted a day figuring this out. */ +type hdr struct { + Count, Slots, Format, Period, Padding uint64 +} + +type record struct { + Count, Size uint64 + Pcs []uint64 +} + +type trailer struct { + Zero, One, Zeroh uint64 +} + +func main() { + var s sample + + r := io.Reader(os.Stdin) + w := io.Writer(os.Stdout) + + records := make(map[string]uint64, 16384) + backtraces := make(map[string][]uint64, 1024) + + /* ignore the documentation, it's wrong, first word must be zero. + * the perl code that figures word length depends on it. + */ + hdr := hdr{0, 3, 0, 10000, 0} + trailer := trailer{0, 1, 0} + start := uint64(0) + end := start + nsamples := end + for binary.Read(r, binary.LittleEndian, &s) == nil { + numpcs := int(s.Wordcount) + bt := make([]uint64, numpcs) + binary.Read(r, binary.LittleEndian, &bt) + //fmt.Printf("%v\n", bt) + record := "" + /* Fix the symbols. pprof was unhappy about the 0xfffffff. + * N.B. The fact that we have to mess with the bt values + * is the reason we did not write a stringer for bt. + */ + for i := range bt { + bt[i] = bt[i] & ((uint64(1) << 32) - 1) + record = record + fmt.Sprintf("0x%x ", bt[i]) + } + records[record]++ + backtraces[record] = bt + //fmt.Printf("%v %d %d %x %v record %v\n", s, s.Wordcount, s.Coreid, s.Ns, bt, record) + /* how sad, once we go to ticks this gets ugly. */ + if start == 0 { + start = s.Ns + } + end = s.Ns + nsamples++ + } + /* we'll need to fix this once we go to ticks. */ + hdr.Period = (end - start) / nsamples + hdr.Count = uint64(0) // !@$@!#$!@#$len(records)) + //fmt.Printf("start %v end %v nsamples %d period %d\n", start, end, nsamples, hdr.Period) + binary.Write(w, binary.LittleEndian, &hdr) + out := make([]uint64, 2) + /* note that the backtrace length varies. But we're good with that. */ + for key, v := range records { + bt := backtraces[key] + out[0] = v + out[1] = uint64(len(bt)) + dump := append(out, bt...) + //fmt.Printf("dump %v\n", dump) + binary.Write(w, binary.LittleEndian, &dump) + } + binary.Write(w, binary.LittleEndian, &trailer) + +} diff --git a/src/jehanne/cmd/preen/.gitignore b/src/jehanne/cmd/preen/.gitignore new file mode 100644 index 0000000..33e5349 --- /dev/null +++ b/src/jehanne/cmd/preen/.gitignore @@ -0,0 +1 @@ +/preen diff --git a/src/jehanne/cmd/preen/preen.go b/src/jehanne/cmd/preen/preen.go new file mode 100644 index 0000000..78a0216 --- /dev/null +++ b/src/jehanne/cmd/preen/preen.go @@ -0,0 +1,157 @@ +package main + +import ( + "encoding/json" + "flag" + "io/ioutil" + "log" + "os" + "path/filepath" +) + +var config = struct { + Jehanne string + Args []string + Except map[string]bool + CmdName string + FullPath string + Src string + Uroot string + Cwd string + Bbsh string + + Goroot string + Gosrcroot string + Arch string + Goos string + Gopath string + TempDir string + Go string + Debug bool + Fail bool +}{ + Jehanne: os.Getenv("JEHANNE"), + Except: map[string]bool{"cflags.json": true, "9": true, "libauthcmd.json": true, "awk.json": true, "bzip2": true, "klibc.json": true, "kcmds.json": true, "klib.json": true, "kernel.json": true}, +} + +type fixup func(string, map[string]interface{}) + +func cflags(n string, jsmap map[string]interface{}) { + if _, ok := jsmap["Cflags"]; ok { + log.Printf("Deleting Cflags from %v", n) + delete(jsmap, "Cflags") + // TODO: once we have another ARCH, use it. + a := []string{"/arch/amd64/include/cflags.json"} + switch tval := jsmap["Include"].(type) { + case []interface{}: + for _, v := range tval { + a = append(a, v.(string)) + } + } + jsmap["Include"] = a + } +} + +func removeempty(n string, jsmap map[string]interface{}) { + for key, val := range jsmap { + switch tval := val.(type) { + case map[string]interface{}: + log.Printf("%s: tval %s", n, tval) + if len(tval) == 0 { + delete(jsmap, key) + } + case []interface{}: + if len(tval) == 0 { + delete(jsmap, key) + } + } + } +} + +func checkname(n string, jsmap map[string]interface{}) { + if _, ok := jsmap["Name"]; !ok { + log.Print("File %v has no \"Name\" key", n) + } +} + +func one(n string, f ...fixup) error { + buf, err := ioutil.ReadFile(n) + if err != nil { + return err + } + + var jsmap map[string]interface{} + if err := json.Unmarshal(buf, &jsmap); err != nil { + return err + } + if config.Debug { + log.Printf("%v: %v", n, jsmap) + } + + for _, d := range f { + d(n, jsmap) + } + buf, err = json.MarshalIndent(jsmap, "", "\t") + if err != nil { + return err + } + buf = append(buf, '\n') + + if config.Debug { + os.Stdout.Write(buf) + } else { + ioutil.WriteFile(n, buf, 0666) + } + return nil +} + +func init() { + if config.Jehanne == "" { + log.Fatalf("Please set $JEHANNE") + } + + if len(config.Args) == 0 { + config.Args = []string{config.Jehanne} + } +} + +func main() { + var err error + + flag.BoolVar(&config.Debug, "d", true, "Enable debug prints") + flag.Parse() + config.Args = flag.Args() + for _, n := range config.Args { + err = filepath.Walk(n, func(name string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if fi.IsDir() { + if config.Except[fi.Name()] { + return filepath.SkipDir + } + return nil + } + n := fi.Name() + if len(n) < 5 || n[len(n)-5:] != ".json" { + return nil + } + todo := []fixup{cflags, removeempty, checkname} + if config.Except[n] { + todo = todo[1:] + } + log.Printf("process %s", name) + err = one(name, todo...) + if err != nil { + log.Printf("%s: %s\n", name, err) + return err + } + return nil + }) + + if err != nil { + log.Fatal("%v", err) + + } + } +} diff --git a/src/jehanne/cmd/profile/profile.go b/src/jehanne/cmd/profile/profile.go new file mode 100644 index 0000000..17573b5 --- /dev/null +++ b/src/jehanne/cmd/profile/profile.go @@ -0,0 +1,131 @@ +package main + +import ( + "bytes" + "debug/elf" + "encoding/binary" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "sort" +) + +type Header struct { + Sz byte + _ byte + Core uint16 + _ uint16 + Ver byte + Sig byte + Time uint64 +} + +type pc uint64 + +type Record struct { + Header + pcs []pc +} + + +type Symbol struct { + Addr uint64 + Name string +} + +type Symbols []*Symbol + +func (s Symbols) Len() int { return len(s) } +func (s Symbols) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type ByAddress struct{ Symbols } + +func (s ByAddress) Less(i, j int) bool { return s.Symbols[i].Addr < s.Symbols[j].Addr } + +var ( + profile = flag.String("profile", "", "name of file containing profile data") + debug = flag.Bool("d", false, "Debug printing") + kernel = flag.String("kernel", "", "kernel to profile against") + symbolTable []*Symbol +) + +func loadSymbols(kernel *elf.File) { + syms, err := kernel.Symbols() + if err != nil { + fmt.Println(err) + return + } + + for _, sym := range syms { + value := sym.Value | 0xffffffff00000000 + symbolTable = append(symbolTable, &Symbol{value, sym.Name}) + } + sort.Sort(ByAddress{symbolTable}) +} + +func findFunction(pc pc) string { + addr := uint64(pc) + var prevSym *Symbol + for _, sym := range symbolTable { + if sym.Addr > addr && prevSym != nil { + return prevSym.Name + } + prevSym = sym + } + return "*** Not Found ***" +} + +func main() { + flag.Parse() + kernelElf, err := elf.Open(*kernel) + if err != nil { + fmt.Println(err) + return + } + loadSymbols(kernelElf) + + backtraces, err := ioutil.ReadFile(*profile) + if err != nil { + fmt.Println(err) + return + } + + b := bytes.NewBuffer(backtraces) + var numRecords int + var records []Record + for { + var h Header + if err := binary.Read(b, binary.LittleEndian, &h); err != nil { + if err == io.EOF { + break + } + fmt.Fprintf(os.Stderr, "header: %v\n", err) + os.Exit(1) + } + if *debug { + fmt.Fprintf(os.Stderr, "%d: %v\n", numRecords, h) + } + + if h.Sig != 0xee { + fmt.Fprintf(os.Stderr, "Record %d: sig is 0x%x, not 0xee", numRecords, h.Sig) + os.Exit(1) + } + pcs := make([]pc, h.Sz) + if err := binary.Read(b, binary.LittleEndian, pcs); err != nil { + fmt.Fprintf(os.Stderr, "data: %v\n", err) + os.Exit(1) + } + numRecords++ + records = append(records, Record{Header: h, pcs: pcs}) + } + + for _, r := range records { + fmt.Printf("0x%x: ", r.Time) + for _, pc := range r.pcs { + fmt.Printf("[0x%x, %s] ", pc, findFunction(pc)) + } + fmt.Printf("\n") + } +} diff --git a/src/jehanne/cmd/removemach/removemach.go b/src/jehanne/cmd/removemach/removemach.go new file mode 100644 index 0000000..0a55f06 --- /dev/null +++ b/src/jehanne/cmd/removemach/removemach.go @@ -0,0 +1,78 @@ +package main + +import ( + "bytes" + "debug/elf" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +var dry = flag.Bool("dryrun", true, "don't really do it") + +func main() { + flag.Parse() + a := flag.Args() + for _, n := range a { + f, err := elf.Open(n) + if err != nil { + fmt.Printf("%v %v\n", n, err) + continue + } + + s, err := f.Symbols() + if err != nil { + fmt.Printf("%v %v\n", n, err) + continue + } + usem := false + for _, v := range s { + if v.Name == "m" && v.Section == elf.SHN_UNDEF { + usem = true + cf := strings.Split(n, ".") + globs, err := filepath.Glob("../*/" + cf[0] + ".c") + if err != nil || len(globs) == 0 { + fmt.Fprintf(os.Stderr, "%v has NO source?\n", cf[0]) + continue + } + if len(globs) > 1 { + fmt.Fprintf(os.Stderr, "Skipping %v has more than one source?\n", cf[0]) + continue + } + file := globs[0] + fi, err := os.Stat(file) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + continue + } + /* OK, read it in, write it out */ + b, err := ioutil.ReadFile(file) + if err != nil { + fmt.Fprintf(os.Stderr, "%v: %v\n", file, err) + } + header := []byte("extern Mach *m; // REMOVE ME\n") + if bytes.Compare(header[:], b[0:len(header)]) == 0 { + fmt.Fprintf(os.Stderr, "%v already done; skipping\n", file) + continue + } + out := append([]byte("typedef struct Mach Mach; extern Mach *m; // REMOVE ME\n"), b...) + if *dry { + fmt.Fprintf(os.Stderr, "Would do %v mode %v\n", file, fi.Mode()) + continue + } + if err := ioutil.WriteFile(file, out, fi.Mode()); err != nil { + fmt.Fprintf(os.Stderr, "Write %v failed: %v; git checkout %v\n", file, err, file) + } + } + } + if !usem { + fmt.Fprintf(os.Stderr, "Ignored %v as it did not reference Mach *m\n", n) + } else { + fmt.Fprintf(os.Stderr, "Procssed %v as it did reference Mach *m\n", n) + } + } + +} diff --git a/src/jehanne/cmd/runqemu/runqemu.go b/src/jehanne/cmd/runqemu/runqemu.go new file mode 100644 index 0000000..327654e --- /dev/null +++ b/src/jehanne/cmd/runqemu/runqemu.go @@ -0,0 +1,119 @@ +// Run commands received from stdin in qemu +// +// -p prompt => prompt to expect (default "10.0.2.15#") +// +// ENVIRONMENT +// Needed: JEHANNE +package main + +import ( + "bufio" + "fmt" + "flag" + "os" + "os/exec" + "strings" + "github.com/kr/pty" + "golang.org/x/crypto/ssh/terminal" +) + +func main() { + var prompt string + flag.StringVar(&prompt, "p", "10.0.2.15#", "the prompt to expect") + flag.Parse() + jehanne := os.Getenv("JEHANNE") + if jehanne == "" { + fmt.Printf("usage: cat cmds.rc | runqemu [-p prompt]\n") + fmt.Printf("error: missing $JEHANNE\n"); + os.Exit(1) + } + if terminal.IsTerminal(0) { + fmt.Printf("usage: cat cmds.rc | runqemu [-p prompt]\n") + fmt.Printf("error: runqemu is intended for automation, pipe commands in.\n") + os.Exit(1) + } + + qemuCmd := "cd $JEHANNE/arch/amd64/kern && $JEHANNE/hacking/QA.sh\n" + qemuCmd = os.ExpandEnv(qemuCmd) + + sh := exec.Command("/bin/sh") + qemu, err := pty.Start(sh) + if err != nil { + fmt.Printf("REGRESS start (%s): %s", qemuCmd, err) + os.Exit(2) + } + qemu.WriteString(qemuCmd) + defer qemu.Close() + + exitStatus := 0 + + qemuInput := make(chan string) + qemuOutputRaw := make(chan string) + wait := make(chan int) + + scanner := bufio.NewScanner(os.Stdin) + + go func() { + qemuOut := make([]byte, 256) + for { + r, err := qemu.Read(qemuOut) + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + wait <- 3 + } + qemuOutputRaw <- string(qemuOut[0:r]) + } + }() + go func(){ + line := "" + for { + s := <- qemuOutputRaw + line += s + if strings.Contains(line, prompt) { + if scanner.Scan() { + cmd := scanner.Text() + qemuInput <- fmt.Sprintf("%s\n", cmd) + } else { + if err := scanner.Err(); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + wait <- 4 + } else { + wait <- exitStatus + } + return + } + fmt.Printf("%s", line) + line = "" + } else if strings.ContainsAny(line, "\r\n") { + if strings.Contains(line, "FAIL") { + exitStatus = 5 + } + fmt.Printf("%s", line) + line = "" + } + } + }() + go func(){ + for { + s := <- qemuInput + i := 0; + for { + n, err := qemu.WriteString(s[i:]) + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + wait <- 6 + } + i += n + if i == len(s) { + break + } + } + } + }() + + e := <- wait + if e == 0 { + fmt.Printf("\nDone.\n") + } + os.Exit(e) +} diff --git a/src/jehanne/cmd/telnet/telnet.go b/src/jehanne/cmd/telnet/telnet.go new file mode 100644 index 0000000..e6e026c --- /dev/null +++ b/src/jehanne/cmd/telnet/telnet.go @@ -0,0 +1,244 @@ +package main + +import ( + "bytes" + "fmt" + "golang.org/x/crypto/ssh/terminal" + "io" + "net" + "os" + "os/signal" + "strings" + "syscall" +) + +// Copyright 2012 the u-root Authors. All rights reserved +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +type ( + chunk struct { + id int + buf []byte + } +) + +const ( + echo = false +) + +func comprefix(lines [][]byte) (int, int) { + var line0 []byte + len0 := 0 + for _, ln := range lines { + if len(ln) > len0 { + line0 = ln + len0 = len(ln) + } + } + ndiff := 0 + for i := range line0 { + nshort := 0 + for _, ln := range lines { + if i >= len(ln) { + nshort++ + } else if ln[i] != line0[i] { + ndiff++ + } + } + if ndiff > 0 { + return i, ndiff + } + if nshort > 0 { + return i, ndiff + } + } + return len(line0), ndiff +} + +func main() { + + if len(os.Args) < 2 { + os.Exit(1) + } + + var conns []*net.TCPConn + for _, a := range os.Args[1:] { + port := "1522" + if !strings.Contains(a, ":") { + a = a + ":" + port + } + + tcpdst, err := net.ResolveTCPAddr("tcp", a) + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } + + c, err := net.DialTCP("tcp", nil, tcpdst) + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } + conns = append(conns, c) + } + + if terminal.IsTerminal(0) { + unraw, err := terminal.MakeRaw(0) + if err != nil { + fmt.Printf("%v\n", err) + os.Exit(1) + } + defer terminal.Restore(0, unraw) + + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, os.Interrupt, os.Kill, syscall.SIGTERM) + go func() { + for _ = range sigc { + terminal.Restore(0, unraw) + os.Exit(1) + } + }() + } + + go func() { + buf := make([]byte, 256) + for { + n, err := os.Stdin.Read(buf) + if err != nil { + if err != io.EOF { + fmt.Printf("%v\n", err) + } + for _, c := range conns { + c.CloseWrite() // I know, I know.. but it is handy sometimes. + } + break + } + buf = buf[0:n] + for i := range buf { + if echo { + fmt.Printf("%s", string(buf[i])) + } + if buf[i] == '\r' { + buf[i] = '\n' + if echo { + fmt.Printf("\n") + } + } + } + + for _, c := range conns { + if _, err := c.Write(buf); err != nil { + fmt.Printf("%v\n", err) + break + } + } + + } + }() + + waitc := make(chan int) + chunkc := make(chan chunk) + for id, c := range conns { + go func(id int, c *net.TCPConn) { + for { + buf := make([]byte, 256) + n, err := c.Read(buf) + if err != nil { + if err != io.EOF { + fmt.Printf("%v\n", err) + } + c.Close() + waitc <- 1 + break + } + if n == 0 { + continue + } + chunkc <- chunk{id, buf[0:n]} + } + }(id, c) + } + + act := make([][]byte, len(conns)) + + go func() { + oldpfix := 0 + insync := true + for { + chunk, more := <-chunkc + if !more { + break + } + id := chunk.id + for _, b := range chunk.buf { + act[id] = append(act[id], b) + if false && b == '\b' { + act[id] = append(act[id], ' ') + act[id] = append(act[id], '\b') + } + } + + /* streams agree, so spit out all they agree on */ + if insync { + newpfix, ndiff := comprefix(act) + if ndiff == 0 && newpfix > oldpfix { + os.Stdout.Write(act[id][oldpfix:newpfix]) + for { + i := bytes.IndexByte(act[id][:newpfix], '\n') + if i == -1 { + break + } + for id := range act { + clen := copy(act[id], act[id][i+1:]) + act[id] = act[id][0:clen] + } + newpfix = newpfix - (i + 1) + } + oldpfix = newpfix + } + + if ndiff > 0 { + insync = false + } + } + + /* + * streams disagree, first to newline completes from oldpfix on. + * the outer loop is just for transitioning from insync, where + * a leader may be several lines ahead + */ + if !insync { + for id := range act { + for { + i := bytes.IndexByte(act[id], '\n') + if i != -1 { + if oldpfix == 0 { + fmt.Fprintf(os.Stdout, "[%d]", id) + } + os.Stdout.Write(act[id][oldpfix : i+1]) + clen := copy(act[id], act[id][i+1:]) + act[id] = act[id][0:clen] + oldpfix = 0 + } else { + break + } + } + } + + /* try for prompt (or sheer luck) */ + if _, ndiff := comprefix(act); ndiff == 0 { + insync = true + oldpfix = 0 + } + } + } + waitc <- 1 + }() + + for _ = range conns { + <-waitc + } + close(chunkc) + <-waitc +} diff --git a/src/jehanne/cmd/vendor/doc.go b/src/jehanne/cmd/vendor/doc.go new file mode 100644 index 0000000..e3a87a0 --- /dev/null +++ b/src/jehanne/cmd/vendor/doc.go @@ -0,0 +1,45 @@ +/* +Vendor is a tool to vendor software for Jehanne. + +It downloads a tarball, verifies it against supplied hashes, extracts it +into "upstream", modifies all the files to be read-only, and then commits +the results. + +When invoked with the flag `-check` it verify that the files present in +a previously vendorized "upstream" folder match those in the downloaded tarball. + +Vendor is purposely unhelpful and un-customisable. + +VENDORFILE + +It requires a "vendor.json" file in the current directory with the following +structure: + + { + "Upstream":"", + "Digest": { + "":"" + }, + "Compress":"", + "RemovePrefix": true, + "Exclude": [ + "" + ] + } + +"Upstream" is the URL to fetch a tarball from. + +"Digest" is a map of algorithm-hash pairs for calculating checksums. All +of the sha functions in the go standard library are supported except for sha1. +The hash is hex-encoded, just like sha*sum output. + +"Compress" is the compression type of the tarball. Gzip and bzip are +supported. If this key is omitted, the tarball is assumed to be uncompressed. + +"RemovePrefix" is a boolean toggle for if the first element of files in the +archive should be removed. Defaults to false if omitted. + +"Exclude" is an array of prefix strings for files that should not be +extracted. They are used as literal prefixes and not interpreted in any way. +*/ +package main diff --git a/src/jehanne/cmd/vendor/vendor.go b/src/jehanne/cmd/vendor/vendor.go new file mode 100644 index 0000000..b5e23d7 --- /dev/null +++ b/src/jehanne/cmd/vendor/vendor.go @@ -0,0 +1,248 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/bzip2" + "compress/gzip" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/json" + "flag" + "hash" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path" + "strings" +) + +const ( + ignore = "*\n!.gitignore\n" + dirPermissions = 0755 +) + +type V struct { + Upstream string + Digest map[string]string + Compress string + RemovePrefix bool + Exclude []string +} + +func main() { + log.SetFlags(log.Lshortfile | log.LstdFlags) + + justCheck := flag.Bool("check", false, "verify the code in upstream/") + + flag.Parse() + + if(*justCheck && !repositoryIsClean()){ + log.Fatal("cannot verify upstream/ files: working directory not clean") + } + + f, err := ioutil.ReadFile("vendor.json") + if err != nil { + log.Fatal(err) + } + + vendor := &V{} + if err := json.Unmarshal(f, vendor); err != nil { + log.Fatal(err) + } + + if _, err := os.Stat("upstream"); err == nil { + log.Println("recreating upstream") + if(*justCheck){ + run("rm", "-r", "-f", "upstream") + } else { + run("git", "rm", "-r", "-f", "upstream") + } + } else { + if(*justCheck){ + log.Fatalf("Cannot verify upstream/ as it does not exists.") + } + os.MkdirAll("patch", dirPermissions) + os.MkdirAll("jehanne", dirPermissions) + ig, err := os.Create(path.Join("jehanne", ".gitignore")) + if err != nil { + log.Fatal(err) + } + defer ig.Close() + if _, err := ig.WriteString(ignore); err != nil { + log.Fatal(err) + } + run("git", "add", ig.Name()) + } + + if err := do(vendor, *justCheck); err != nil { + log.Fatal(err) + } + + if(*justCheck){ + if(repositoryIsClean()){ + log.Printf("the files in upstream/ matches those in "+path.Base(vendor.Upstream)) + } else { + log.Fatalf("the files in upstream/ does not match those in "+path.Base(vendor.Upstream)) + } + } else { + run("git", "add", "vendor.json") + run("git", "commit", "-s", "-m", "vendor: pull in "+path.Base(vendor.Upstream)) + } +} + +func repositoryIsClean() bool { + out, err := exec.Command("git", "status", "--porcelain").Output() + if err != nil { + log.Fatal(err) + } + if(len(out) > 0){ + log.Println("git status --porcelain"); + log.Println(string(out)) + return false + } + return true +} + +func do(v *V, justCheck bool) error { + name := fetch(v) + f, err := os.Open(name) + if err != nil { + return err + } + defer os.Remove(name) + + var unZ io.Reader + switch v.Compress { + case "gzip": + unZ, err = gzip.NewReader(f) + if err != nil { + return err + } + case "bzip2": + unZ = bzip2.NewReader(f) + default: + unZ = f + } + + ar := tar.NewReader(unZ) + h, err := ar.Next() +untar: + for ; err == nil; h, err = ar.Next() { + n := h.Name + if v.RemovePrefix { + n = strings.SplitN(n, "/", 2)[1] + } + for _, ex := range v.Exclude { + if strings.HasPrefix(n, ex) { + continue untar + } + } + n = path.Join("upstream", n) + if h.FileInfo().IsDir() { + os.MkdirAll(n, dirPermissions) + continue + } + os.MkdirAll(path.Dir(n), dirPermissions) + out, err := os.Create(n) + if err != nil { + log.Println(err) + continue + } + + if n, err := io.Copy(out, ar); n != h.Size || err != nil { + return err + } + out.Close() + } + if err != io.EOF { + return err + } + + if(justCheck){ + return nil; + } + return run("git", "add", "upstream") +} + +type match struct { + hash.Hash + Good []byte + Name string +} + +func (m match) OK() bool { + return bytes.Equal(m.Good, m.Hash.Sum(nil)) +} + +func fetch(v *V) string { + if len(v.Digest) == 0 { + log.Fatal("no checksums specifed") + } + + f, err := ioutil.TempFile("", "cmdVendor") + if err != nil { + log.Fatal(err) + } + defer f.Close() + + req, err := http.NewRequest("GET", v.Upstream, nil) + if err != nil { + log.Fatal(err) + } + client := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DisableCompression: true, + }, + } + res, err := client.Do(req) + if err != nil { + log.Fatal(err) + } + defer res.Body.Close() + + var digests []match + for k, v := range v.Digest { + g, err := hex.DecodeString(v) + if err != nil { + log.Fatal(err) + } + switch k { + case "sha224": + digests = append(digests, match{sha256.New224(), g, k}) + case "sha256": + digests = append(digests, match{sha256.New(), g, k}) + case "sha384": + digests = append(digests, match{sha512.New384(), g, k}) + case "sha512": + digests = append(digests, match{sha512.New(), g, k}) + } + } + ws := make([]io.Writer, len(digests)) + for i := range digests { + ws[i] = digests[i] + } + w := io.MultiWriter(ws...) + + if _, err := io.Copy(f, io.TeeReader(res.Body, w)); err != nil { + log.Fatal(err) + } + for _, h := range digests { + if !h.OK() { + log.Fatalf("mismatched %q hash\n\tWanted %x\n\tGot %x\n", h.Name, h.Good, h.Hash.Sum(nil)) + } + } + return f.Name() +} + +func run(exe string, arg ...string) error { + cmd := exec.Command(exe, arg...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +}