diff --git a/.gitignore b/.gitignore index fd882f4105..a1f6ed16a2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /lr +mqlr dist alpine-container.tar centos-container.tar diff --git a/providers-sdk/v1/mqlr/cmd/docs.go b/providers-sdk/v1/mqlr/cmd/docs.go index 354281345a..8f75e8a239 100644 --- a/providers-sdk/v1/mqlr/cmd/docs.go +++ b/providers-sdk/v1/mqlr/cmd/docs.go @@ -9,7 +9,6 @@ import ( "os" "path" "path/filepath" - "sort" "strings" "text/template" @@ -59,74 +58,6 @@ var docsYamlCmd = &cobra.Command{ }, } -// required to be before more detail platform to ensure the right mapping -var platformMappingKeys = []string{ - "aws", "gcp", "k8s", "azure", "azurerm", "arista", "equinix", "ms365", "msgraph", "vsphere", "esxi", "terraform", "terraform.state", "terraform.plan", -} - -var platformMapping = map[string][]string{ - "aws": {"aws"}, - "gcp": {"gcp"}, - "k8s": {"kubernetes"}, - "azure": {"azure"}, - "azurerm": {"azure"}, - "arista": {"arista-eos"}, - "equinix": {"equinix"}, - "ms365": {"microsoft365"}, - "msgraph": {"microsoft365"}, - "vsphere": {"vmware-esxi", "vmware-vsphere"}, - "esxi": {"vmware-esxi", "vmware-vsphere"}, - "terraform": {"terraform-hcl"}, - "terraform.state": {"terraform-state"}, - "terraform.plan": {"terraform-plan"}, -} - -func ensureDefaults(id string, entry *lrcore.LrDocsEntry, version string) *lrcore.LrDocsEntry { - for _, k := range platformMappingKeys { - if entry == nil { - entry = &lrcore.LrDocsEntry{} - } - if entry.MinMondooVersion == "" { - entry.MinMondooVersion = version - } else if entry.MinMondooVersion == defaultVersionField && version != defaultVersionField { - // Update to specified version if previously set to default - entry.MinMondooVersion = version - } - if strings.HasPrefix(id, k) { - entry.Platform = &lrcore.LrDocsPlatform{ - Name: platformMapping[k], - } - } - } - return entry -} - -func mergeFields(version string, entry *lrcore.LrDocsEntry, fields []*lrcore.BasicField) { - if entry == nil && len(fields) > 0 { - entry = &lrcore.LrDocsEntry{} - entry.Fields = map[string]*lrcore.LrDocsField{} - } else if entry == nil { - return - } else if entry.Fields == nil { - entry.Fields = map[string]*lrcore.LrDocsField{} - } - docFields := entry.Fields - for _, f := range fields { - if docFields[f.ID] == nil { - fDoc := &lrcore.LrDocsField{ - MinMondooVersion: version, - } - entry.Fields[f.ID] = fDoc - } else if entry.Fields[f.ID].MinMondooVersion == defaultVersionField && version != defaultVersionField { - entry.Fields[f.ID].MinMondooVersion = version - } - // Scrub field version if same as resource - if entry.Fields[f.ID].MinMondooVersion == entry.MinMondooVersion { - entry.Fields[f.ID].MinMondooVersion = "" - } - } -} - var docsJsonCmd = &cobra.Command{ Use: "json", Short: "convert yaml docs manifest into json", @@ -157,43 +88,6 @@ func runDocsYamlCmd(lrFile string, headerFile string, version string, docsFilePa return } - // to ensure we generate the same markdown, we sort the resources first - sort.SliceStable(res.Resources, func(i, j int) bool { - return res.Resources[i].ID < res.Resources[j].ID - }) - - d := lrcore.LrDocs{ - Resources: map[string]*lrcore.LrDocsEntry{}, - } - - fields := map[string][]*lrcore.BasicField{} - isPrivate := map[string]bool{} - for i := range res.Resources { - id := res.Resources[i].ID - isPrivate[id] = res.Resources[i].IsPrivate - d.Resources[id] = nil - if res.Resources[i].Body != nil { - basicFields := []*lrcore.BasicField{} - for _, f := range res.Resources[i].Body.Fields { - if f.BasicField != nil { - basicFields = append(basicFields, f.BasicField) - } - } - fields[id] = basicFields - } - } - - // default behaviour is to output the result on cli - if docsFilePath == "" { - data, err := yaml.Marshal(d) - if err != nil { - log.Fatal().Err(err).Msg("could not marshal docs") - } - - fmt.Println(string(data)) - return - } - // if an file was provided, we check if the file exist and merge existing content with the new resources // to ensure that existing documentation stays available var existingData lrcore.LrDocs @@ -208,28 +102,28 @@ func runDocsYamlCmd(lrFile string, headerFile string, version string, docsFilePa if err != nil { log.Fatal().Err(err).Msg("could not load yaml data") } + } - log.Info().Msg("merge content") - for k := range existingData.Resources { - v := existingData.Resources[k] - d.Resources[k] = v - } + docs, err := res.GenerateDocs(version, defaultVersionField, existingData) + if err != nil { + log.Fatal().Err(err).Msg("could not generate docs") } + // default behaviour is to output the result on cli + if docsFilePath == "" { + data, err := yaml.Marshal(docs) + if err != nil { + log.Fatal().Err(err).Msg("could not marshal docs") + } - // ensure default values and fields are set - for k := range d.Resources { - d.Resources[k] = ensureDefaults(k, d.Resources[k], version) - mergeFields(version, d.Resources[k], fields[k]) - // Merge in other doc fields from core.lr - d.Resources[k].IsPrivate = isPrivate[k] + fmt.Println(string(data)) + return } // generate content - data, err := yaml.Marshal(d) + data, err := yaml.Marshal(docs) if err != nil { log.Fatal().Err(err).Msg("could not marshal docs") } - // add license header var headerTpl *template.Template if headerFile != "" { diff --git a/providers-sdk/v1/mqlr/cmd/docs_test.go b/providers-sdk/v1/mqlr/cmd/docs_test.go deleted file mode 100644 index 685b2092aa..0000000000 --- a/providers-sdk/v1/mqlr/cmd/docs_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Mondoo, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package cmd - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "go.mondoo.com/cnquery/v12/providers-sdk/v1/mqlr/lrcore" -) - -var defaultLrDocsEntry = &lrcore.LrDocsEntry{ - Fields: map[string]*lrcore.LrDocsField{}, - MinMondooVersion: "9.1.0", -} - -func TestPlatformMapping(t *testing.T) { - res := ensureDefaults("terraform.plan.configuration", defaultLrDocsEntry, "9.1.0") - assert.Equal(t, "terraform-plan", res.Platform.Name[0]) - - res = ensureDefaults("terraform.plan.proposedChange", defaultLrDocsEntry, "9.1.0") - assert.Equal(t, "terraform-plan", res.Platform.Name[0]) - - res = ensureDefaults("terraform.state.module", defaultLrDocsEntry, "9.1.0") - assert.Equal(t, "terraform-state", res.Platform.Name[0]) - - res = ensureDefaults("terraform.block", defaultLrDocsEntry, "9.1.0") - assert.Equal(t, "terraform-hcl", res.Platform.Name[0]) -} diff --git a/providers-sdk/v1/mqlr/cmd/root.go b/providers-sdk/v1/mqlr/cmd/root.go index ee8d910f13..8cc1dc15ca 100644 --- a/providers-sdk/v1/mqlr/cmd/root.go +++ b/providers-sdk/v1/mqlr/cmd/root.go @@ -15,6 +15,15 @@ import ( var cfgFile string +func init() { + cobra.OnInitialize(initConfig) + + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "Set config file path (default $HOME/.lr.yaml)") +} + // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "cli", @@ -34,15 +43,6 @@ func Execute() { } } -func init() { - cobra.OnInitialize(initConfig) - - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "Set config file path (default $HOME/.lr.yaml)") -} - // initConfig reads in config file and ENV variables if set. func initConfig() { if cfgFile != "" { diff --git a/providers-sdk/v1/mqlr/lrcore/docs.go b/providers-sdk/v1/mqlr/lrcore/docs.go index ebf6154728..4b59773026 100644 --- a/providers-sdk/v1/mqlr/lrcore/docs.go +++ b/providers-sdk/v1/mqlr/lrcore/docs.go @@ -5,6 +5,8 @@ package lrcore import ( "fmt" + "maps" + "sort" "strconv" "strings" @@ -23,7 +25,7 @@ type LrDocsEntry struct { Platform *LrDocsPlatform `json:"platform,omitempty"` Docs *LrDocsDocumentation `json:"docs,omitempty"` Resources []LrDocsRefs `json:"resources,omitempty"` - Fields map[string]*LrDocsField `json:"fields,omitEmpty"` + Fields map[string]*LrDocsField `json:"fields,omitempty"` Refs []LrDocsRefs `json:"refs,omitempty"` Snippets []LrDocsSnippet `json:"snippets,omitempty"` IsPrivate bool `json:"is_private,omitempty"` @@ -96,3 +98,111 @@ func InjectMetadata(schema *resources.Schema, docs *LrDocs) { } } } + +func (lr *LR) GenerateDocs(currentVersion, defaultVersion string, existingDocs LrDocs) (LrDocs, error) { + // to ensure we generate the same markdown, we sort the resources first + sort.SliceStable(lr.Resources, func(i, j int) bool { + return lr.Resources[i].ID < lr.Resources[j].ID + }) + + docs := LrDocs{Resources: map[string]*LrDocsEntry{}} + + fields := map[string][]*BasicField{} + isPrivate := map[string]bool{} + for i := range lr.Resources { + id := lr.Resources[i].ID + isPrivate[id] = lr.Resources[i].IsPrivate + docs.Resources[id] = nil + if lr.Resources[i].Body != nil { + basicFields := []*BasicField{} + for _, f := range lr.Resources[i].Body.Fields { + if f.BasicField != nil { + basicFields = append(basicFields, f.BasicField) + } + } + fields[id] = basicFields + } + } + + // if we have docs from existing manifest, merge them in + if existingDocs.Resources != nil { + maps.Copy(docs.Resources, existingDocs.Resources) + } + // ensure default values and fields are set + for k := range docs.Resources { + docs.Resources[k] = ensureDefaults(k, docs.Resources[k], currentVersion, defaultVersion) + mergeFields(docs.Resources[k], fields[k], currentVersion, defaultVersion) + // Merge in other doc fields from core.lr + docs.Resources[k].IsPrivate = isPrivate[k] + } + + return docs, nil +} + +func mergeFields(entry *LrDocsEntry, fields []*BasicField, currentVersion, defaultVersion string) { + if entry == nil && len(fields) > 0 { + entry = &LrDocsEntry{} + entry.Fields = map[string]*LrDocsField{} + } else if entry == nil { + return + } else if entry.Fields == nil { + entry.Fields = map[string]*LrDocsField{} + } + docFields := entry.Fields + for _, f := range fields { + if docFields[f.ID] == nil { + fDoc := &LrDocsField{ + MinMondooVersion: currentVersion, + } + entry.Fields[f.ID] = fDoc + } else if entry.Fields[f.ID].MinMondooVersion == defaultVersion && currentVersion != defaultVersion { + entry.Fields[f.ID].MinMondooVersion = currentVersion + } + // Scrub field version if same as resource + if entry.Fields[f.ID].MinMondooVersion == entry.MinMondooVersion { + entry.Fields[f.ID].MinMondooVersion = "" + } + } +} + +func ensureDefaults(id string, entry *LrDocsEntry, currentVersion, defaultVersion string) *LrDocsEntry { + for _, k := range platformMappingKeys { + if entry == nil { + entry = &LrDocsEntry{} + } + if entry.MinMondooVersion == "" { + entry.MinMondooVersion = currentVersion + } else if entry.MinMondooVersion == defaultVersion && currentVersion != defaultVersion { + // Update to specified version if previously set to default + entry.MinMondooVersion = currentVersion + } + if strings.HasPrefix(id, k) { + entry.Platform = &LrDocsPlatform{ + Name: platformMapping[k], + } + } + } + return entry +} + +// required to be before more detail platform to ensure the right mapping +var platformMappingKeys = []string{ + "aws", "gcp", "k8s", "azure", "azurerm", "arista", "equinix", "ms365", "msgraph", "vsphere", "esxi", "terraform", "terraform.state", "terraform.plan", +} + +var platformMapping = map[string][]string{ + "aws": {"aws"}, + "gcp": {"gcp"}, + "k8s": {"kubernetes"}, + "azure": {"azure"}, + "azurerm": {"azure"}, + "arista": {"arista-eos"}, + "equinix": {"equinix"}, + "ms365": {"microsoft365"}, + "msgraph": {"microsoft365"}, + "vsphere": {"vmware-esxi", "vmware-vsphere"}, + "esxi": {"vmware-esxi", "vmware-vsphere"}, + "terraform": {"terraform-hcl"}, + "terraform.state": {"terraform-state"}, + "terraform.plan": {"terraform-plan"}, +} diff --git a/providers-sdk/v1/mqlr/lrcore/docs_test.go b/providers-sdk/v1/mqlr/lrcore/docs_test.go new file mode 100644 index 0000000000..8a8e65a2e9 --- /dev/null +++ b/providers-sdk/v1/mqlr/lrcore/docs_test.go @@ -0,0 +1,60 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package lrcore + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/yaml" +) + +var defaultLrDocsEntry = &LrDocsEntry{ + Fields: map[string]*LrDocsField{}, + MinMondooVersion: "9.1.0", +} + +func TestPlatformMapping(t *testing.T) { + res := ensureDefaults("terraform.plan.configuration", defaultLrDocsEntry, "9.1.0", "9.1.0") + assert.Equal(t, "terraform-plan", res.Platform.Name[0]) + + res = ensureDefaults("terraform.plan.proposedChange", defaultLrDocsEntry, "9.1.0", "9.1.0") + assert.Equal(t, "terraform-plan", res.Platform.Name[0]) + + res = ensureDefaults("terraform.state.module", defaultLrDocsEntry, "9.1.0", "9.1.0") + assert.Equal(t, "terraform-state", res.Platform.Name[0]) + + res = ensureDefaults("terraform.block", defaultLrDocsEntry, "9.1.0", "9.1.0") + assert.Equal(t, "terraform-hcl", res.Platform.Name[0]) +} + +func TestGenerateDocs(t *testing.T) { + lrFile, err := os.ReadFile("testdata/new.lr") + assert.NoError(t, err) + existingDocs, err := os.ReadFile("testdata/existing-manifest.yaml") + assert.NoError(t, err) + var lrDocsData LrDocs + err = yaml.Unmarshal(existingDocs, &lrDocsData) + assert.NoError(t, err) + lr := parse(t, string(lrFile)) + + docs, err := lr.GenerateDocs("9.1.0", "9.1.0", lrDocsData) + assert.NoError(t, err) + assert.NotNil(t, docs) + assert.Len(t, docs.Resources, 3) + + names := []string{} + for k := range docs.Resources { + names = append(names, k) + } + expected := []string{"sshd", "sshd.config", "auditd.config"} + assert.ElementsMatch(t, expected, names) + + // verify min version for existing resources hasnt changed + assert.Equal(t, "5.15.0", docs.Resources["sshd"].MinMondooVersion) + assert.Equal(t, "5.15.0", docs.Resources["sshd.config"].MinMondooVersion) + // new resource has the default min version + assert.Equal(t, "9.1.0", docs.Resources["auditd.config"].MinMondooVersion) +} diff --git a/providers-sdk/v1/mqlr/lrcore/testdata/existing-manifest.yaml b/providers-sdk/v1/mqlr/lrcore/testdata/existing-manifest.yaml new file mode 100644 index 0000000000..95a1c45ade --- /dev/null +++ b/providers-sdk/v1/mqlr/lrcore/testdata/existing-manifest.yaml @@ -0,0 +1,23 @@ +resources: + sshd: + fields: {} + min_mondoo_version: 5.15.0 + sshd.config: + fields: + blocks: + min_mondoo_version: latest + ciphers: {} + content: {} + file: {} + files: + min_mondoo_version: latest + hostkeys: {} + kexs: {} + macs: {} + params: {} + permitRootLogin: + min_mondoo_version: latest + min_mondoo_version: 5.15.0 + snippets: + - query: sshd.config.params['Banner'] == '/etc/ssh/sshd-banner' + title: Check that the SSH banner is sourced from /etc/ssh/sshd-banner \ No newline at end of file diff --git a/providers-sdk/v1/mqlr/lrcore/testdata/new.lr b/providers-sdk/v1/mqlr/lrcore/testdata/new.lr new file mode 100644 index 0000000000..4c03930d47 --- /dev/null +++ b/providers-sdk/v1/mqlr/lrcore/testdata/new.lr @@ -0,0 +1,21 @@ +sshd {} + +sshd.config { + init(path? string) + file() file + files(file) []file + content(file) string + params(file) map[string]string + blocks(file) []sshd.config.matchBlock + ciphers(params) []string + macs(params) []string + kexs(params) []string + hostkeys(params) []string + permitRootLogin(params) []string +} + +auditd.config { + init(path? string) + file() file + params(file) map[string]string +} \ No newline at end of file