From f8fb88816a04eba8170260eab75de4169d9077be Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 17 Feb 2020 10:11:16 -0800 Subject: [PATCH] matrix is done Signed-off-by: Casey Lee --- cmd/root.go | 2 +- pkg/common/cartesian.go | 54 ++++++++++++++++++++++++ pkg/common/cartesian_test.go | 28 +++++++++++++ pkg/model/workflow.go | 3 ++ pkg/runner/run_context.go | 65 ++++++++++++++++++++++------- pkg/runner/runner.go | 65 +++++++++++++++++++++++------ pkg/runner/step.go | 46 +++++++++++++------- pkg/runner/testdata/matrix/push.yml | 2 +- 8 files changed, 219 insertions(+), 46 deletions(-) create mode 100644 pkg/common/cartesian.go create mode 100644 pkg/common/cartesian_test.go diff --git a/cmd/root.go b/cmd/root.go index ddb71bc..4b05c79 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -62,7 +62,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str var eventName string if len(args) > 0 { eventName = args[0] - } else if events := planner.GetEvents(); len(events) > 1 { + } else if events := planner.GetEvents(); len(events) > 0 { // set default event type to first event // this way user dont have to specify the event. log.Debugf("Using detected workflow event: %s", events[0]) diff --git a/pkg/common/cartesian.go b/pkg/common/cartesian.go new file mode 100644 index 0000000..a4485a0 --- /dev/null +++ b/pkg/common/cartesian.go @@ -0,0 +1,54 @@ +package common + +// CartesianProduct takes map of lists and returns list of unique tuples +func CartesianProduct(mapOfLists map[string][]interface{}) []map[string]interface{} { + listNames := make([]string, 0) + lists := make([][]interface{}, 0) + for k, v := range mapOfLists { + listNames = append(listNames, k) + lists = append(lists, v) + } + + listCart := cartN(lists...) + + rtn := make([]map[string]interface{}, 0) + for _, list := range listCart { + vMap := make(map[string]interface{}) + for i, v := range list { + vMap[listNames[i]] = v + } + rtn = append(rtn, vMap) + } + return rtn +} + +func cartN(a ...[]interface{}) [][]interface{} { + c := 1 + for _, a := range a { + c *= len(a) + } + if c == 0 { + return nil + } + p := make([][]interface{}, c) + b := make([]interface{}, c*len(a)) + n := make([]int, len(a)) + s := 0 + for i := range p { + e := s + len(a) + pi := b[s:e] + p[i] = pi + s = e + for j, n := range n { + pi[j] = a[j][n] + } + for j := len(n) - 1; j >= 0; j-- { + n[j]++ + if n[j] < len(a[j]) { + break + } + n[j] = 0 + } + } + return p +} diff --git a/pkg/common/cartesian_test.go b/pkg/common/cartesian_test.go new file mode 100644 index 0000000..07450f5 --- /dev/null +++ b/pkg/common/cartesian_test.go @@ -0,0 +1,28 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCartisianProduct(t *testing.T) { + assert := assert.New(t) + input := map[string][]interface{}{ + "foo": []interface{}{1, 2, 3, 4}, + "bar": []interface{}{"a", "b", "c"}, + "baz": []interface{}{false, true}, + } + + output := CartesianProduct(input) + assert.Len(output, 24) + + for _, v := range output { + assert.Len(v, 3) + + assert.Contains(v, "foo") + assert.Contains(v, "bar") + assert.Contains(v, "baz") + } + +} diff --git a/pkg/model/workflow.go b/pkg/model/workflow.go index 6989df9..c25cf5c 100644 --- a/pkg/model/workflow.go +++ b/pkg/model/workflow.go @@ -187,6 +187,9 @@ func ReadWorkflow(in io.Reader) (*Workflow, error) { func (w *Workflow) GetJob(jobID string) *Job { for id, j := range w.Jobs { if jobID == id { + if j.Name == "" { + j.Name = id + } return j } } diff --git a/pkg/runner/run_context.go b/pkg/runner/run_context.go index b823d3d..7aea62a 100644 --- a/pkg/runner/run_context.go +++ b/pkg/runner/run_context.go @@ -24,16 +24,16 @@ import ( // RunContext contains info about current job type RunContext struct { - Config *Config - Matrix map[string]interface{} - Run *model.Run - EventJSON string - Env map[string]string - Tempdir string - ExtraPath []string - CurrentStep string - StepResults map[string]*stepResult - PlatformName string + Config *Config + Matrix map[string]interface{} + Run *model.Run + EventJSON string + Env map[string]string + Tempdir string + ExtraPath []string + CurrentStep string + StepResults map[string]*stepResult + ExprEval ExpressionEvaluator } type stepResult struct { @@ -56,9 +56,6 @@ func (rc *RunContext) Close(ctx context.Context) error { // Executor returns a pipeline executor for all the steps in the job func (rc *RunContext) Executor() common.Executor { - if img := platformImage(rc.PlatformName); img == "" { - return common.NewInfoExecutor(" \U0001F6A7 Skipping unsupported platform '%s'", rc.PlatformName) - } err := rc.setupTempDir() if err != nil { @@ -77,6 +74,13 @@ func (rc *RunContext) Executor() common.Executor { Success: true, Outputs: make(map[string]string), } + rc.ExprEval = rc.NewStepExpressionEvaluator(s) + + if !rc.EvalBool(s.If) { + log.Debugf("Skipping step '%s' due to '%s'", s.String(), s.If) + return nil + } + common.Logger(ctx).Infof("\u2B50 Run %s", s) err := rc.newStepExecutor(s)(ctx) if err == nil { @@ -88,7 +92,36 @@ func (rc *RunContext) Executor() common.Executor { return err }) } - return common.NewPipelineExecutor(steps...).Finally(rc.Close) + return func(ctx context.Context) error { + defer rc.Close(ctx) + job := rc.Run.Job() + log := common.Logger(ctx) + if !rc.EvalBool(job.If) { + log.Debugf("Skipping job '%s' due to '%s'", job.Name, job.If) + return nil + } + + platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) + if img := platformImage(platformName); img == "" { + log.Infof(" \U0001F6A7 Skipping unsupported platform '%s'", platformName) + return nil + } + + return common.NewPipelineExecutor(steps...)(ctx) + } +} + +// EvalBool evaluates an expression against current run context +func (rc *RunContext) EvalBool(expr string) bool { + if expr != "" { + v, err := rc.ExprEval.Evaluate(expr) + if err != nil { + log.Errorf("Error evaluating expression '%s' - %v", expr, err) + return false + } + return v == "true" + } + return true } func mergeMaps(maps ...map[string]string) map[string]string { @@ -141,10 +174,10 @@ func (rc *RunContext) runContainer(containerSpec *model.ContainerSpec) common.Ex } var cmd, entrypoint []string if containerSpec.Args != "" { - cmd = strings.Fields(containerSpec.Args) + cmd = strings.Fields(rc.ExprEval.Interpolate(containerSpec.Args)) } if containerSpec.Entrypoint != "" { - entrypoint = strings.Fields(containerSpec.Entrypoint) + entrypoint = strings.Fields(rc.ExprEval.Interpolate(containerSpec.Entrypoint)) } rawLogger := common.Logger(ctx).WithField("raw_output", true) diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 5b37855..73169a0 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -53,18 +53,49 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { for _, stage := range plan.Stages { stageExecutor := make([]common.Executor, 0) for _, run := range stage.Runs { - // TODO - don't just grab first index of each dimension - matrix := make(map[string]interface{}) - if run.Job().Strategy != nil { - for mkey, mvals := range run.Job().Strategy.Matrix { - if mkey == "include" || mkey == "exclude" { - continue - } - matrix[mkey] = mvals[0] + job := run.Job() + matrixes := make([]map[string]interface{}, 0) + if job.Strategy != nil { + includes := make([]map[string]interface{}, 0) + for _, v := range job.Strategy.Matrix["include"] { + includes = append(includes, v.(map[string]interface{})) } + delete(job.Strategy.Matrix, "include") + + excludes := make([]map[string]interface{}, 0) + for _, v := range job.Strategy.Matrix["exclude"] { + excludes = append(excludes, v.(map[string]interface{})) + } + delete(job.Strategy.Matrix, "exclude") + + matrixProduct := common.CartesianProduct(job.Strategy.Matrix) + + MATRIX: + for _, matrix := range matrixProduct { + for _, exclude := range excludes { + if commonKeysMatch(matrix, exclude) { + log.Debugf("Skipping matrix '%v' due to exclude '%v'", matrix, exclude) + continue MATRIX + } + } + for _, include := range includes { + if commonKeysMatch(matrix, include) { + log.Debugf("Setting add'l values on matrix '%v' due to include '%v'", matrix, include) + for k, v := range include { + matrix[k] = v + } + } + } + matrixes = append(matrixes, matrix) + } + + } else { + matrixes = append(matrixes, make(map[string]interface{})) } - stageExecutor = append(stageExecutor, runner.NewRunExecutor(run, matrix)) + for _, matrix := range matrixes { + stageExecutor = append(stageExecutor, runner.NewRunExecutor(run, matrix)) + } } pipeline = append(pipeline, common.NewParallelExecutor(stageExecutor...)) } @@ -72,6 +103,15 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { return common.NewPipelineExecutor(pipeline...) } +func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool { + for aKey, aVal := range a { + if bVal, ok := b[aKey]; ok && aVal != bVal { + return false + } + } + return true +} + func (runner *runnerImpl) NewRunExecutor(run *model.Run, matrix map[string]interface{}) common.Executor { rc := new(RunContext) rc.Config = runner.config @@ -79,11 +119,12 @@ func (runner *runnerImpl) NewRunExecutor(run *model.Run, matrix map[string]inter rc.EventJSON = runner.eventJSON rc.StepResults = make(map[string]*stepResult) rc.Matrix = matrix - - ee := rc.NewExpressionEvaluator() - rc.PlatformName = ee.Interpolate(run.Job().RunsOn) + rc.ExprEval = rc.NewExpressionEvaluator() return func(ctx context.Context) error { ctx = WithJobLogger(ctx, rc.Run.String()) + if len(rc.Matrix) > 0 { + common.Logger(ctx).Infof("\U0001F9EA Matrix: %v", rc.Matrix) + } return rc.Executor()(ctx) } } diff --git a/pkg/runner/step.go b/pkg/runner/step.go index d8c7800..80258b8 100644 --- a/pkg/runner/step.go +++ b/pkg/runner/step.go @@ -15,17 +15,33 @@ import ( log "github.com/sirupsen/logrus" ) +func (rc *RunContext) StepEnv(step *model.Step) map[string]string { + var env map[string]string + job := rc.Run.Job() + if job.Container != nil { + env = mergeMaps(rc.GetEnv(), job.Container.Env, step.GetEnv()) + } else { + env = mergeMaps(rc.GetEnv(), step.GetEnv()) + } + + for k, v := range env { + env[k] = rc.ExprEval.Interpolate(v) + } + return env +} + +func (rc *RunContext) setupEnv(containerSpec *model.ContainerSpec, step *model.Step) common.Executor { + return func(ctx context.Context) error { + containerSpec.Env = rc.withGithubEnv(rc.StepEnv(step)) + return nil + } +} + func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { - ee := rc.NewStepExpressionEvaluator(step) job := rc.Run.Job() containerSpec := new(model.ContainerSpec) - containerSpec.Env = rc.withGithubEnv(rc.StepEnv(step)) containerSpec.Name = rc.createContainerName(step.ID) - for k, v := range containerSpec.Env { - containerSpec.Env[k] = ee.Interpolate(v) - } - switch step.Type() { case model.StepTypeRun: if job.Container != nil { @@ -34,9 +50,11 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { containerSpec.Volumes = job.Container.Volumes containerSpec.Options = job.Container.Options } else { - containerSpec.Image = platformImage(rc.PlatformName) + platformName := rc.ExprEval.Interpolate(rc.Run.Job().RunsOn) + containerSpec.Image = platformImage(platformName) } return common.NewPipelineExecutor( + rc.setupEnv(containerSpec, step), rc.setupShellCommand(containerSpec, step.Shell, step.Run), rc.pullImage(containerSpec), rc.runContainer(containerSpec), @@ -47,6 +65,7 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { containerSpec.Entrypoint = step.With["entrypoint"] containerSpec.Args = step.With["args"] return common.NewPipelineExecutor( + rc.setupEnv(containerSpec, step), rc.pullImage(containerSpec), rc.runContainer(containerSpec), ) @@ -54,6 +73,7 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { case model.StepTypeUsesActionLocal: containerSpec.Image = fmt.Sprintf("%s:%s", containerSpec.Name, "latest") return common.NewPipelineExecutor( + rc.setupEnv(containerSpec, step), rc.setupAction(containerSpec, filepath.Join(rc.Config.Workdir, step.Uses)), applyWith(containerSpec, step), rc.pullImage(containerSpec), @@ -78,6 +98,7 @@ func (rc *RunContext) newStepExecutor(step *model.Step) common.Executor { Ref: remoteAction.Ref, Dir: cloneDir, }), + rc.setupEnv(containerSpec, step), rc.setupAction(containerSpec, filepath.Join(cloneDir, remoteAction.Path)), applyWith(containerSpec, step), rc.pullImage(containerSpec), @@ -100,15 +121,6 @@ func applyWith(containerSpec *model.ContainerSpec, step *model.Step) common.Exec } } -// StepEnv returns the env for a step -func (rc *RunContext) StepEnv(step *model.Step) map[string]string { - job := rc.Run.Job() - if job.Container != nil { - return mergeMaps(rc.GetEnv(), job.Container.Env, step.GetEnv()) - } - return mergeMaps(rc.GetEnv(), step.GetEnv()) -} - func (rc *RunContext) setupShellCommand(containerSpec *model.ContainerSpec, shell string, run string) common.Executor { return func(ctx context.Context) error { shellCommand := "" @@ -140,6 +152,8 @@ func (rc *RunContext) setupShellCommand(containerSpec *model.ContainerSpec, shel return err } + run = rc.ExprEval.Interpolate(run) + if _, err := tempScript.WriteString(run); err != nil { return err } diff --git a/pkg/runner/testdata/matrix/push.yml b/pkg/runner/testdata/matrix/push.yml index c31d2ba..c1dc793 100644 --- a/pkg/runner/testdata/matrix/push.yml +++ b/pkg/runner/testdata/matrix/push.yml @@ -5,7 +5,7 @@ jobs: build: runs-on: ${{ matrix.os }} steps: - - run: echo ${NODE_VERSION} | grep 4 + - run: echo ${NODE_VERSION} | grep ${{ matrix.node }} env: NODE_VERSION: ${{ matrix.node }} strategy: