Somehow these tests have been passing but shouldn't have been

Signed-off-by: John McCrae <[email protected]>

Author: johnmccrae

oc-chef-pedant/Gemfile

@@ -14,7 +14,8 @@ gem "rest-client", git: "https://github.com/chef/rest-client", branch: "jfm/ucrt
 # gem "chef", "18.8.46" - commmenting this out to get past loading that enormous gem to only get a version number.
 
 # For "rake chef_zero_spec"
-# gem 'chef-zero', github: 'chef/chef-zero'
+# PR #352 merged to main but not released yet - use main branch
+gem 'chef-zero', github: 'chef/chef-zero', branch: 'main'
 
 # If you want to load debugging tools into the bundle exec sandbox,
 # # add these additional dependencies into Gemfile.local

oc-chef-pedant/Gemfile.lock

@@ -37,15 +37,15 @@ GEM
     binding_of_caller (1.0.1)
       debug_inspector (>= 1.2.0)
     byebug (12.0.0)
-    chef-utils (18.8.46)
+    chef-utils (18.8.54)
       concurrent-ruby
     coderay (1.1.3)
     concurrent-ruby (1.3.5)
     debug_inspector (1.2.0)
-    diff-lcs (1.5.1)
+    diff-lcs (1.6.2)
     domain_name (0.6.20240107)
     erubis (2.7.0)
-    ffi (1.16.3)
+    ffi (1.17.2-x64-mingw-ucrt)
     ffi-win32-extensions (1.0.4)
       ffi
     http-accept (2.1.1)
@@ -60,8 +60,6 @@ GEM
     mixlib-authentication (3.0.10)
     mixlib-config (3.0.27)
       tomlrb
-    mixlib-shellout (3.3.9)
-      chef-utils
     mixlib-shellout (3.3.9-x64-mingw-ucrt)
       chef-utils
       ffi-win32-extensions (~> 1.0.3)
@@ -79,7 +77,7 @@ GEM
       binding_of_caller (~> 1.0)
       pry (~> 0.13)
     public_suffix (6.0.2)
-    rake (13.3.0)
+    rake (13.3.1)
     rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -89,24 +87,22 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.6)
+    rspec-mocks (3.13.7)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-rerun (1.1.0)
       rspec (~> 3.0)
     rspec-support (3.13.6)
     rspec_junit_formatter (0.6.0)
       rspec-core (>= 2, < 4, != 2.12.0)
-    tomlrb (1.3.0)
+    tomlrb (2.0.3)
     uuidtools (2.2.0)
     win32-process (0.10.0)
       ffi (>= 1.0.0)
     wmi-lite (1.0.7)
 
 PLATFORMS
-  ruby
   x64-mingw-ucrt
-  x86_64-linux
 
 DEPENDENCIES
   oc-chef-pedant!
@@ -117,4 +113,4 @@ DEPENDENCIES
   rest-client!
 
 BUNDLED WITH
-   2.6.9
+   2.7.2

oc-chef-pedant/Rakefile

@@ -15,7 +15,7 @@ task :chef_zero_spec do
 end
 
 def bundle_exec_with_chef(test_gem, commands)
-  gem_path = Bundler.environment.specs[test_gem].first.full_gem_path
+  gem_path = Bundler.load.specs[test_gem].first.full_gem_path
   @gemfile_path = File.join(gem_path, "Gemfile.#{CURRENT_GEM_NAME}-external-test")
   gemfile = File.open(@gemfile_path, "w")
   begin

oc-chef-pedant/TESTING_GUIDE.md

@@ -0,0 +1,152 @@
+# oc-chef-pedant Testing Guide
+
+This guide standardizes how to run the oc-chef-pedant suite across environments (chef-server vs chef-zero) and documents recent helper scope changes, risk notes, and the validation matrix post ActiveSupport removal.
+
+---
+
+## 1. Supported Execution Modes
+
+| Mode | Target | Purpose | Invocation Summary |
+|------|--------|---------|--------------------|
+| Server (Embedded) | Full Chef Server (dev VM / omnibus install) | Primary regression signal | `chef-server-ctl test` (inside VM) or `bin/oc-chef-pedant -c pedant_config.rb` |
+| chef-zero (Released Gem) | Upstream chef-zero gem matching Chef 18 baseline | Fast API contract smoke, isolate server-side regressions | Rake task / direct script (see below) |
+| chef-zero (Branch Override) | Branch removing legacy Chef::VERSION references | Validate removal of ActiveSupport & version coupling | Export `CHEF_VERSION` or Bundler override Gemfile |
+
+> Ruby 3.1.7 is the reference version (Chef 18 parity). Avoid 3.4+ until broader compatibility is validated.
+
+---
+
+## 2. Quick Commands
+
+### 2.1 Inside Dev VM (full server)
+
+```bash
+chef-server-ctl test --all            # Full pedant + internal org tests
+chef-server-ctl test                  # Default subset
+chef-server-ctl test --all --exclude-internal-orgs
+```
+
+### 2.2 Local (without full server) using chef-zero
+
+From `oc-chef-pedant/`:
+
+```bash
+# Ensure bundle resolves to the released chef-zero
+bundle install
+rake chef_zero_spec
+```
+
+### 2.3 Using Branch Override of chef-zero
+
+Use a dedicated Gemfile (example: `scripts/bk_tests/chef_zero-Gemfile`) or add a block like:
+
+```ruby
+# Gemfile.local snippet
+# gem 'chef-zero', git: 'https://github.com/chef/chef-zero', branch: 'remove-chef-version'
+```
+
+Then:
+
+```bash
+BUNDLE_GEMFILE=scripts/bk_tests/chef_zero-Gemfile bundle install
+BUNDLE_GEMFILE=scripts/bk_tests/chef_zero-Gemfile rake chef_zero_spec
+```
+
+### 2.4 Focused Spec Samples
+
+```bash
+bundle exec rspec spec/api/keys/user_keys_spec.rb:930-970
+bundle exec rspec spec/api/server_api_version_spec.rb
+```
+
+---
+
+## 3. Helper Scope Changes (IMPORTANT)
+
+Removed class-level singleton helpers:
+
+- `platform`
+- `api_url`
+
+Rationale: They caused large clusters of WrongScope failures when indirectly invoked in `before(:all)` and polluted example group scope. Use patterns below:
+
+| Context | Old (removed) | New (explicit) |
+|---------|---------------|----------------|
+| before(:all)/after(:all) | `platform.create_user(name)` | `Pedant::Config.pedant_platform.create_user(name)` |
+| Example / let / before(:each) | `platform` (still OK via instance helper) | `platform` |
+| Building URLs (group scope) | `api_url("/nodes")` | `Pedant::Config.pedant_platform.api_url("/nodes")` |
+
+Instance helpers remain: `platform`, `api_url(path)` inside example scope only.
+
+---
+
+## 4. Validation Matrix
+
+| Dimension | Set | Notes |
+|-----------|-----|-------|
+| Ruby | 3.1.7 | Canonical; matches Chef 18 runtime |
+| Platform Mode | Full Server, chef-zero (released), chef-zero (branch) | Branch mode only when validating coupling removal |
+| Spec Slice | Smoke (server_api_version_spec + a CRUD resource), Core (default), Full (`--all`) | Smoke runs should complete < 2 min on typical dev hardware |
+| Headers | `X-Chef-Version` hardcoded KNIFE_VERSION | No dynamic Chef::VERSION dependency |
+| API Versions | v0 (legacy), negotiation path (`/server_api_version`) | Ensure `server_api_version_spec` passes in all modes |
+
+Minimum CI gates after recent changes:
+
+1. Server Mode Core suite: PASS
+2. chef-zero Released Smoke suite: PASS (server_api_version + basic ACL or keys spec subset)
+3. Branch Override (only while feature branch active): Smoke suite PASS
+4. Full Server `--all` (nightly) PASS
+
+---
+
+## 5. Risk & Compatibility Notes
+
+| Area | Risk | Mitigation |
+|------|------|------------|
+| Helper Scope Removal | Hidden reliance on class-level `platform` in `before(:all)` blocks causing NameError later | Grep for `before(:all)` + `platform.`; replace with explicit config reference |
+| ActiveSupport Removal | Missed implicit inflector / Hash key conversions | Added targeted replacements (`pedant/concern`, `stringify_keys` shim) & core suite green before broader matrix |
+| Version Decoupling | Tools expecting dynamic Chef::VERSION in headers | Hardcoded `KNIFE_VERSION` unchanged; document expectation |
+| chef-zero Branch Divergence | Drift from released gem causing false positives | Keep branch usage confined to feature PR validation; do not gate merges on branch-only passes |
+| Ruby Version Skew | Newer Ruby 3.3/3.4 differences (warning categories, RSpec timing) | Pin CI to 3.1.7; add optional future lane once core stabilized |
+
+Rollback Strategy: Re-introducing class-level helpers is low complexity but discouraged; prefer updating any lingering specs. ActiveSupport dependency should not be restored—issues should be fixed by adding narrowly scoped utilities.
+
+---
+
+## 6. Troubleshooting Quick Table
+
+| Symptom | Likely Cause | Action |
+|---------|--------------|--------|
+| 1000+ WrongScope / method missing `platform` failures | A `before(:all)` still calling instance helper | Replace with `Pedant::Config.pedant_platform` |
+| 401/403 explosion under chef-zero only | Org/user provisioning order or missing association | Re-run single spec with `--backtrace`, inspect platform creation logs |
+| All requests 500 in chef-zero mode | Wrong chef-zero branch / dependency mismatch | Confirm Gemfile lock; run `ruby -v`; re-bundle with released gem |
+| Version negotiation failures | Header mismatch / test expecting dynamic version | Inspect `X-Chef-Version` in captured request log |
+
+---
+
+## 7. Adding New Tests – Best Practices
+
+- Use `let` for per-example dynamic data; use `shared(:symbol)` only when a value must be accessible inside `before(:all)`.
+- Avoid creating real users/orgs in `before(:all)` unless they are immutable and shared; prefer `before(:each)` for mutable objects.
+- Always clean up with explicit `Pedant::Config.pedant_platform.delete_*` calls in `after(:all)` for created actors.
+- For new endpoints: add smoke coverage to the Smoke slice (small curated set) to keep early failures tight.
+
+---
+
+## 8. Future Enhancements (Optional Backlog)
+
+- Introduce a tagged `:smoke` subset with < 50 examples for <60s validation.
+- Add script to auto-rewrite lingering `platform.` calls in group hooks.
+- Parameterize KNIFE_VERSION via env var for experimental clients.
+
+---
+
+## 9. At-a-Glance Checklist (PR Author)
+
+- [ ] Core suite passes locally (server mode)
+- [ ] Smoke suite passes with released chef-zero
+- [ ] No new `before(:all)` using instance helpers
+- [ ] Added/updated docs if new config flags introduced
+
+---
+Last updated: 2025-11-06

oc-chef-pedant/lib/pedant/rspec/common.rb

@@ -37,6 +37,18 @@ module Common
         include Pedant::JSON
         include Pedant::Request
         include Pedant::RSpec::CommonResponses
+        
+        # NOTE: Previously we defined class-level helpers `platform` and `api_url` so they
+        # could be called from before(:all)/after(:all). This encouraged mixing example-
+        # scoped state into group-level hooks and produced large waves of WrongScope style
+        # failures under RSpec 3 when helpers were (indirectly) used during example group
+        # definition. We intentionally remove those singleton methods; only instance-level
+        # helpers remain. For before(:all)/after(:all) use
+        #   Pedant::Config.pedant_platform
+        # or
+        #   Pedant::Config.pedant_platform.api_url("/path")
+        # explicitly. This makes scope usage explicit and prevents accidental memoization
+        # leakage across examples.
 
 
         # Request/Response
@@ -405,20 +417,22 @@ def once_per_run(key, &block)
           Pedant::OncePerRun.once_per_run(key, &block)
         end
 
+        # platform and api_url helper methods
+        # These instance-level helpers are accessible from examples, let blocks,
+        # before(:each), and after(:each). In before(:all)/after(:all) blocks, use
+        # Pedant::Config.pedant_platform directly to avoid RSpec scope issues.
         def platform
           Pedant::Config.pedant_platform
         end
 
+        def api_url(path_fragment)
+          Pedant::Config.pedant_platform.api_url(path_fragment)
+        end
+        
         ## TODO: Remove this method; we probably don't need to access it directly
         def server
-          platform.server
-        end
-
-        # construct a complete API URL based on the pre-configured server information
-        def api_url(path_fragment)
-          platform.api_url(path_fragment)
+          platform
         end
-
         # Given a response object, verify the HTTP status is in the
         # 200-ish success range and raise an error if it is not. This
         # is intended to be used in helper/util modules where we want
@@ -469,21 +483,21 @@ def set_opt(opt, default)
         ################################################################################
 
         # If these are referenced in before(:all) blocks, use shared() instead of let()
-        shared(:organization)     { platform.test_org }
-        shared(:org)              { platform.test_org.name }
+        shared(:organization)     { Pedant::Config.pedant_platform.test_org }
+        shared(:org)              { Pedant::Config.pedant_platform.test_org.name }
         # TODO look at how these are set up - is accurate?
-        shared(:admin_user)       { platform.admin_user }
-        shared(:org_admin)        { platform.admin_user }
-        shared(:normal_user)      { platform.non_admin_user }
+        shared(:admin_user)       { Pedant::Config.pedant_platform.admin_user }
+        shared(:org_admin)        { Pedant::Config.pedant_platform.admin_user }
+        shared(:normal_user)      { Pedant::Config.pedant_platform.non_admin_user }
 
-        shared(:outside_user)     { platform.bad_user }
+        shared(:outside_user)     { Pedant::Config.pedant_platform.bad_user }
 
         # TODO no such thing - eliminate tests referring to it!
-        shared(:admin_client)     { platform.admin_client }
+        shared(:admin_client)     { Pedant::Config.pedant_platform.admin_client }
         # TODO all non-validator clients are normal clients.
-        shared(:normal_client)    { platform.non_admin_client }
-        shared(:outside_client)   { platform.bad_client }
-        shared(:validator_client) { platform.validator_client }
+        shared(:normal_client)    { Pedant::Config.pedant_platform.non_admin_client }
+        shared(:outside_client)   { Pedant::Config.pedant_platform.bad_client }
+        shared(:validator_client) { Pedant::Config.pedant_platform.validator_client }
 
         shared(:knife_admin)      { admin_user }
         shared(:knife_user)       { normal_user }
@@ -492,18 +506,18 @@ def set_opt(opt, default)
         # away, and all tasks that require its use become methods on the
         # Platform object
 
-        shared(:superuser) { platform.superuser }
-        shared(:superuser_key) { platform.superuser_key }
-        shared(:webui_key) { platform.webui_key }
+        shared(:superuser) { Pedant::Config.pedant_platform.superuser }
+        shared(:superuser_key) { Pedant::Config.pedant_platform.superuser_key }
+        shared(:webui_key) { Pedant::Config.pedant_platform.webui_key }
 
         # Given a requestor, create a new one with the same name, but
         # with the web UI's private key for 'impersonation' tests
         def impersonate(requestor_to_impersonate)
-          Pedant::Requestor.new(requestor_to_impersonate.name, platform.webui_key)
+          Pedant::Requestor.new(requestor_to_impersonate.name, Pedant::Config.pedant_platform.webui_key)
         end
 
         # Need a well-formed yet invalid key for a requestor to test authentiction
-        shared(:bogus_key) { platform.bogus_key }
+        shared(:bogus_key) { Pedant::Config.pedant_platform.bogus_key }
         shared(:invalid_user) { Pedant::Requestor.new("invalid", bogus_key, bogus: true) }
 
         ################################################################################

oc-chef-pedant/spec/api/account/account_acl_spec.rb

@@ -17,11 +17,11 @@ def rand_id
 
   before(:all) do
     @test_orgname2 = "test-org-#{rand_id}"
-    platform.create_org(@test_orgname2) if Pedant.config[:org][:create_me]
+    Pedant::Config.pedant_platform.create_org(@test_orgname2) if Pedant.config[:org][:create_me]
   end
 
   after(:all) do
-    platform.delete_org(@test_orgname2) if Pedant.config[:org][:create_me]
+    Pedant::Config.pedant_platform.delete_org(@test_orgname2) if Pedant.config[:org][:create_me]
   end
 
   context "/users/<name>/_acl endpoint" do
@@ -1283,11 +1283,11 @@ def rand_id
 
             context "PUT /#{type}/<name>/_acl/#{permission}" do
               before(:all) do
-                @client_with_dot = platform.create_client("test-client.#{rand_id}")
+                @client_with_dot = Pedant::Config.pedant_platform.create_client("test-client.#{rand_id}")
               end
 
               after(:all) do
-                platform.delete_client(@client_with_dot)
+                Pedant::Config.pedant_platform.delete_client(@client_with_dot)
               end
 
 

oc-chef-pedant/spec/api/account/account_group_spec.rb

@@ -440,11 +440,11 @@
 
     before(:all) do
       @test_orgname2 = "test-org-#{rand_id}"
-      platform.create_org(@test_orgname2) if Pedant.config[:org][:create_me]
+      Pedant::Config.pedant_platform.create_org(@test_orgname2) if Pedant.config[:org][:create_me]
     end
 
     after(:all) do
-      platform.delete_org(@test_orgname2) if Pedant.config[:org][:create_me]
+      Pedant::Config.pedant_platform.delete_org(@test_orgname2) if Pedant.config[:org][:create_me]
     end
 
     before :each do