Commit dc5854e1 by Ben Clayton

Regres: Further optimizations for coverage

Compress common spans into groups and index these. Reduces the full coverage json from ~500MB to just under 100MB. Use zlib compression on the final file instead of a zip. The compression algorithm is the same, and produces roughly the same sized output, but a raw zlib file is much quicker to decompress with pako instead of jszip. Bug: b/152192800 Change-Id: Id444ddf4027b6ace03570719f159ae14e481cd37 Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/43249 Kokoro-Presubmit: kokoro <noreply+kokoro@google.com> Reviewed-by: 's avatarAntonio Maiorano <amaiorano@google.com> Tested-by: 's avatarBen Clayton <bclayton@google.com>
parent c6e7b2a4
......@@ -26,14 +26,12 @@
package main
import (
"archive/zip"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"math"
......@@ -759,33 +757,23 @@ func (r *regres) commitCoverage(cov *cov.Tree, revision git.Hash) error {
dir := filepath.Join(r.cacheRoot, "coverage")
defer os.RemoveAll(dir)
if err := git.CheckoutRemoteBranch(dir, url, coverageBranch); err != nil {
return fmt.Errorf("Failed to checkout gh-pages branch: %v", err)
return cause.Wrap(err, "Failed to checkout gh-pages branch")
}
filePath := filepath.Join(dir, "coverage.zip")
filePath := filepath.Join(dir, "coverage.dat")
file, err := os.Create(filePath)
if err != nil {
return fmt.Errorf("Failed to create file '%s': %v", filePath, err)
return cause.Wrap(err, "Failed to create file '%s'", filePath)
}
defer file.Close()
coverage := cov.JSON(revision.String())
zw := zip.NewWriter(file)
zfw, err := zw.Create("coverage.json")
if err != nil {
return fmt.Errorf("Failed to create 'coverage.json' file in zip: %v", err)
}
if _, err := io.Copy(zfw, strings.NewReader(coverage)); err != nil {
return fmt.Errorf("Failed to compress coverage datas: %v", err)
}
if err := zw.Close(); err != nil {
return fmt.Errorf("Failed to close zip file: %v", err)
if err := cov.Encode(revision.String(), file); err != nil {
return cause.Wrap(err, "Failed to encode coverage")
}
file.Close()
if err := git.Add(dir, filePath); err != nil {
return fmt.Errorf("Failed to git add '%s': %v", filePath, err)
return cause.Wrap(err, "Failed to git add '%s'", filePath)
}
shortHash := revision.String()[:8]
......@@ -795,18 +783,17 @@ func (r *regres) commitCoverage(cov *cov.Tree, revision git.Hash) error {
Email: r.gerritEmail,
})
if err != nil {
return fmt.Errorf("Failed to 'git commit': %v", err)
return cause.Wrap(err, "Failed to git commit")
}
if !r.dryRun {
err = git.Push(dir, url, coverageBranch, coverageBranch, git.PushFlags{})
if err != nil {
return fmt.Errorf("Failed to 'git push': %v", err)
return cause.Wrap(err, "Failed to 'git push'")
}
log.Printf("Coverage for %v pushed to Github\n", shortHash)
}
log.Printf("Coverage for %v pushed to Github\n", shortHash)
return nil
}
......
......@@ -36,6 +36,7 @@ import (
"strings"
"time"
"../../cause"
"../../cov"
"../../deqp"
"../../llvm"
......@@ -132,8 +133,12 @@ func run() error {
}
if *genCoverage {
if err := ioutil.WriteFile("coverage.json", []byte(res.Coverage.JSON("master")), 0666); err != nil {
return err
f, err := os.Create("coverage.dat")
if err != nil {
return cause.Wrap(err, "Couldn't open coverage.dat file")
}
if err := res.Coverage.Encode("master", f); err != nil {
return cause.Wrap(err, "Couldn't encode coverage data")
}
}
......
......@@ -225,7 +225,7 @@ func checkTests(t *testing.T, tree *cov.Tree, expect string) {
}
func checkCoverage(t *testing.T, tree *cov.Tree, file string, expect string) {
g, e := tree.File(file).String(tree.Tests(), tree.Strings()), expect
g, e := tree.FileCoverage(file).String(tree.Tests(), tree.Strings()), expect
if tg, te := trimWS(g), trimWS(e); tg != te {
t.Errorf("Coverage not as expected.\nGot:\n%v\nExpect:\n%v\n------\nGot: %v\nExpect: %v", g, e, tg, te)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment