Maybe/lib/tasks/benchmarking.rake

# Benchmarking requires a production-like data sample, so requires some up-front setup.
#
# 1. Load a scrubbed production-like slice of data into maybe_benchmarking DB locally
# 2. Setup .env.production so that the Rails app can boot with RAILS_ENV=production and connect to local maybe_benchmarking DB
# 3. Run `rake benchmark_dump:06_setup_bench_user`
# 4. Run locally, find endpoint needed
# 5. Run an endpoint, example: `ENDPOINT=/budgets/jun-2025/budget_categories/245637cb-129f-4612-b0a8-1de57559372b RAILS_ENV=production BENCHMARKING_ENABLED=true RAILS_LOG_LEVEL=debug rake benchmarking:ips`
namespace :benchmarking do
  desc "Shorthand task for running warm/cold benchmark"
  task endpoint: :environment do
    system(
      "RAILS_ENV=production BENCHMARKING_ENABLED=true ENDPOINT=#{ENV.fetch("ENDPOINT", "/")} rake benchmarking:warm_cold_endpoint_ips"
    )
  end

  # When to use: Track overall endpoint speed improvements over time (recommended, most practical test)
  desc "Run cold & warm performance benchmarks and append to history"
  task warm_cold_endpoint_ips: :environment do
    path = ENV.fetch("ENDPOINT", "/")

    # 🚫 Fail fast unless the benchmark is run in production mode
    unless Rails.env.production?
      raise "benchmark:ips must be run with RAILS_ENV=production (current: #{Rails.env})"
    end

    # ---------------------------------------------------------------------------
    # Tunable parameters – override with environment variables if needed
    # ---------------------------------------------------------------------------
    cold_warmup     = Integer(ENV.fetch("COLD_WARMUP", 0))  # seconds to warm up before *cold* timing (0 == true cold)
    cold_iterations = Integer(ENV.fetch("COLD_ITERATIONS", 1)) # requests to measure for the cold run

    warm_warmup     = Integer(ENV.fetch("WARM_WARMUP", 5))  # seconds benchmark-ips uses to stabilise JIT/caches
    warm_time       = Integer(ENV.fetch("WARM_TIME", 10))   # seconds benchmark-ips samples for warm statistics
    # ---------------------------------------------------------------------------

    setup_benchmark_env(path)
    FileUtils.mkdir_p("tmp/benchmarks")

    timestamp  = Time.current.strftime("%Y-%m-%d %H:%M:%S")
    commit_sha = `git rev-parse --short HEAD 2>/dev/null`.strip rescue "unknown"
    puts "🕒 Starting benchmark run at #{timestamp} (#{commit_sha})"

    # 🚿  Flush application caches so the first request is a *true* cold hit
    Rails.cache&.clear if defined?(Rails)

    # ---------------------------
    # 1️⃣  Cold measurement
    # ---------------------------
    puts "❄️  Running cold benchmark for #{path} (#{cold_iterations} iteration)..."
    cold_cmd = "IPS_WARMUP=#{cold_warmup} IPS_TIME=0 IPS_ITERATIONS=#{cold_iterations} " \
               "bundle exec derailed exec perf:ips"
    cold_output = `#{cold_cmd} 2>&1`

    puts "Cold output:"
    puts cold_output

    cold_result = extract_clean_results(cold_output)

    # ---------------------------
    # 2️⃣  Warm measurement
    # ---------------------------
    puts "🔥 Running warm benchmark for #{path} (#{warm_time}s sample)..."
    warm_cmd = "IPS_WARMUP=#{warm_warmup} IPS_TIME=#{warm_time} " \
               "bundle exec derailed exec perf:ips"
    warm_output = `#{warm_cmd} 2>&1`

    puts "Warm output:"
    puts warm_output

    warm_result = extract_clean_results(warm_output)

    # ---------------------------------------------------------------------------
    # Persist results
    # ---------------------------------------------------------------------------
    separator        = "\n" + "=" * 70 + "\n"
    timestamp_header = "#{separator}📊 BENCHMARK RUN - #{timestamp} (#{commit_sha})#{separator}"

    # Table header
    table_header    = "| Type | IPS | Deviation | Time/Iteration | Iterations | Total Time |\n"
    table_separator = "|------|-----|-----------|----------------|------------|------------|\n"

    cold_row        = format_table_row("COLD", cold_result)
    warm_row        = format_table_row("WARM", warm_result)

    combined_result = table_header + table_separator + cold_row + warm_row + "\n"

    File.open(benchmark_file(path), "a") { |f| f.write(timestamp_header + combined_result) }

    puts "✅ Results saved to #{benchmark_file(path)}"
  end

  private
    def setup_benchmark_env(path)
      ENV["USE_AUTH"]      = "true"
      ENV["USE_SERVER"]    = "puma"
      ENV["PATH_TO_HIT"]   = path
      ENV["HTTP_METHOD"]   = "GET"
      ENV["RAILS_LOG_LEVEL"] ||= "error" # keep output clean
    end

    def benchmark_file(path)
      filename = case path
      when "/" then "dashboard"
      else
        path.gsub("/", "_").gsub(/^_+/, "")
      end
      "tmp/benchmarks/#{filename}.txt"
    end

    def extract_clean_results(output)
      lines = output.split("\n")

      # Example benchmark-ips output line:
      # "         SomeLabel    14.416k (± 3.8%) i/s -     72.000k in   5.004618s"
      result_line = lines.find { |line| line.match(/\d[\d\.kM]*\s+\(±\s*[0-9\.]+%\)\s+i\/s/) }

      if result_line
        if (match = result_line.match(/(\d[\d\.kM]*)\s+\(±\s*([0-9\.]+)%\)\s+i\/s\s+(?:\(([^)]+)\)\s+)?-\s+(\d[\d\.kM]*)\s+in\s+(\d+\.\d+)s/))
          ips_value          = match[1]
          deviation_percent  = match[2].to_f
          time_per_iteration = match[3] || "-"
          iterations         = match[4]
          total_time         = "#{match[5]}s"

          {
            ips:                ips_value,
            deviation:          "± %.2f%%" % deviation_percent,
            time_per_iteration: time_per_iteration,
            iterations:         iterations,
            total_time:         total_time
          }
        else
          no_data_hash
        end
      else
        no_data_hash("No results")
      end
    end

    def format_table_row(type, data)
      # Wider deviation column accommodates strings like "± 0.12%"
      "| %-4s | %-5s | %-11s | %-14s | %-10s | %-10s |\n" % [
        type,
        data[:ips],
        data[:deviation],
        data[:time_per_iteration],
        data[:iterations],
        data[:total_time]
      ]
    end

    def no_data_hash(ips_msg = "No data")
      {
        ips:                ips_msg,
        deviation:          "-",
        time_per_iteration: "-",
        iterations:         "-",
        total_time:         "-"
      }
    end
end