157dbbb9eb
## Phase 8: Enterprise Extractor Improvements ✅ - 14 security extractors (TLS, JWT, SQL injection, XSS, etc.) - 10 framework-specific extractors (Spring, Django, Rails, etc.) - Config file security detection (YAML, TOML) ## Phase 9: Autonomous Extractor Generation ✅ - Shadow mode executor with TP/FP tracking - Graduation pipeline with confidence thresholds - Auto-rollback on regression detection - Cross-project pattern syncing ## UAT Suite Complete (14 scripts, 90 tests) - test-core-detection.sh (6 tests) - test-declarative-extractors.sh (5 tests) - test-domain-frameworks.sh (5 tests) - test-domain-unreal.sh (3 tests) - test-llm-extraction.sh (6 tests) - test-eval-harness.sh (5 tests) - test-cross-language.sh (3 tests) - test-precommit-performance.sh (4 tests) - test-output-formats.sh (8 tests) - test-drift-detection.sh (6 tests) - test-exit-codes.sh (12 tests) + 3 more scripts ## Other Changes - Updated roadmap to mark Phase 8-9 complete - Added .gitignore entries for build artifacts - Updated pre-commit: 800 line limit, exclude tests/data/cmd Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
452 lines
12 KiB
Bash
Executable File
452 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# test-domain-frameworks.sh - Validate framework-specific security extractors
|
|
# Part of the Comprehensive Vision UAT
|
|
|
|
set -euo pipefail
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
UAT_DIR="$(dirname "$SCRIPT_DIR")"
|
|
APHORIA_DIR="$(dirname "$UAT_DIR")"
|
|
STEMEDB_DIR="$(dirname "$(dirname "$APHORIA_DIR")")"
|
|
|
|
# Build Aphoria if needed
|
|
APHORIA_BIN="${STEMEDB_DIR}/target/release/aphoria"
|
|
if [[ ! -f "$APHORIA_BIN" ]]; then
|
|
echo "Building Aphoria..."
|
|
cargo build --release --package aphoria --manifest-path "${STEMEDB_DIR}/Cargo.toml"
|
|
fi
|
|
|
|
# Test fixtures directory
|
|
FIXTURES_DIR="${UAT_DIR}/fixtures"
|
|
mkdir -p "$FIXTURES_DIR"
|
|
|
|
PASSED=0
|
|
FAILED=0
|
|
TOTAL=0
|
|
|
|
test_case() {
|
|
local id="$1"
|
|
local description="$2"
|
|
TOTAL=$((TOTAL + 1))
|
|
echo -e "\n${YELLOW}[$id]${NC} $description"
|
|
}
|
|
|
|
pass() {
|
|
PASSED=$((PASSED + 1))
|
|
echo -e " ${GREEN}✓ PASS${NC}"
|
|
}
|
|
|
|
fail() {
|
|
local reason="$1"
|
|
FAILED=$((FAILED + 1))
|
|
echo -e " ${RED}✗ FAIL: $reason${NC}"
|
|
}
|
|
|
|
# Create test fixtures for each framework
|
|
create_fixtures() {
|
|
echo "Creating framework security test fixtures..."
|
|
|
|
# Django fixtures
|
|
mkdir -p "${FIXTURES_DIR}/django"
|
|
cat > "${FIXTURES_DIR}/django/pyproject.toml" << 'EOF'
|
|
[project]
|
|
name = "django-test"
|
|
version = "0.1.0"
|
|
EOF
|
|
cat > "${FIXTURES_DIR}/django/settings.py" << 'EOF'
|
|
# Django settings file
|
|
from django.conf import settings
|
|
|
|
DEBUG = True
|
|
ALLOWED_HOSTS = ['*']
|
|
SECRET_KEY = 'insecure-development-key'
|
|
SESSION_COOKIE_SECURE = False
|
|
CSRF_COOKIE_SECURE = False
|
|
EOF
|
|
|
|
# Flask fixtures
|
|
mkdir -p "${FIXTURES_DIR}/flask"
|
|
cat > "${FIXTURES_DIR}/flask/pyproject.toml" << 'EOF'
|
|
[project]
|
|
name = "flask-test"
|
|
version = "0.1.0"
|
|
EOF
|
|
cat > "${FIXTURES_DIR}/flask/app.py" << 'EOF'
|
|
from flask import Flask
|
|
|
|
app = Flask(__name__)
|
|
app.debug = True
|
|
app.config['WTF_CSRF_ENABLED'] = False
|
|
app.secret_key = 'dev'
|
|
|
|
@app.route('/')
|
|
def index():
|
|
return 'Hello'
|
|
|
|
if __name__ == '__main__':
|
|
app.run(debug=True)
|
|
EOF
|
|
|
|
# Spring fixtures
|
|
mkdir -p "${FIXTURES_DIR}/spring"
|
|
cat > "${FIXTURES_DIR}/spring/SecurityConfig.java" << 'EOF'
|
|
package com.example.security;
|
|
|
|
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
|
|
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
|
|
|
|
@EnableWebSecurity
|
|
public class SecurityConfig extends WebSecurityConfigurerAdapter {
|
|
@Override
|
|
protected void configure(HttpSecurity http) throws Exception {
|
|
http.csrf().disable();
|
|
http.authorizeRequests()
|
|
.antMatchers("/**").permitAll();
|
|
http.headers().frameOptions().disable();
|
|
}
|
|
}
|
|
EOF
|
|
|
|
# Express fixtures
|
|
mkdir -p "${FIXTURES_DIR}/express"
|
|
cat > "${FIXTURES_DIR}/express/package.json" << 'EOF'
|
|
{
|
|
"name": "express-test",
|
|
"version": "1.0.0",
|
|
"main": "server.js"
|
|
}
|
|
EOF
|
|
cat > "${FIXTURES_DIR}/express/server.js" << 'EOF'
|
|
const express = require('express');
|
|
const cors = require('cors');
|
|
const session = require('express-session');
|
|
|
|
const app = express();
|
|
|
|
// BAD: CORS with wildcard origin and credentials
|
|
app.use(cors({
|
|
origin: '*',
|
|
credentials: true
|
|
}));
|
|
|
|
app.use(session({
|
|
secret: 'keyboard cat',
|
|
resave: false,
|
|
cookie: {
|
|
secure: false,
|
|
httpOnly: false
|
|
}
|
|
}));
|
|
|
|
app.listen(3000);
|
|
EOF
|
|
|
|
# Rails fixtures
|
|
mkdir -p "${FIXTURES_DIR}/rails/config/environments"
|
|
mkdir -p "${FIXTURES_DIR}/rails/app/controllers"
|
|
cat > "${FIXTURES_DIR}/rails/config/environments/production.rb" << 'EOF'
|
|
Rails.application.configure do
|
|
config.force_ssl = false
|
|
config.log_level = :debug
|
|
end
|
|
EOF
|
|
cat > "${FIXTURES_DIR}/rails/app/controllers/api_controller.rb" << 'EOF'
|
|
class ApiController < ApplicationController
|
|
skip_before_action :verify_authenticity_token
|
|
protect_from_forgery with: :null_session
|
|
|
|
def search
|
|
User.where("name = '#{params[:name]}'")
|
|
end
|
|
end
|
|
EOF
|
|
|
|
# FastAPI fixtures
|
|
mkdir -p "${FIXTURES_DIR}/fastapi"
|
|
cat > "${FIXTURES_DIR}/fastapi/pyproject.toml" << 'EOF'
|
|
[project]
|
|
name = "fastapi-test"
|
|
version = "0.1.0"
|
|
EOF
|
|
cat > "${FIXTURES_DIR}/fastapi/main.py" << 'EOF'
|
|
from fastapi import FastAPI
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
app = FastAPI(debug=True)
|
|
|
|
# BAD: CORS with wildcard and credentials
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
)
|
|
|
|
SECRET_KEY = "hardcoded-secret"
|
|
|
|
@app.get("/")
|
|
def read_root():
|
|
return {"Hello": "World"}
|
|
EOF
|
|
|
|
# Laravel fixtures
|
|
mkdir -p "${FIXTURES_DIR}/laravel/app/Http"
|
|
cat > "${FIXTURES_DIR}/laravel/.env" << 'EOF'
|
|
APP_NAME=Laravel
|
|
APP_ENV=production
|
|
APP_KEY=
|
|
APP_DEBUG=true
|
|
SESSION_SECURE_COOKIE=false
|
|
EOF
|
|
cat > "${FIXTURES_DIR}/laravel/app/Http/UserController.php" << 'EOF'
|
|
<?php
|
|
namespace App\Http\Controllers;
|
|
|
|
use Illuminate\Http\Request;
|
|
use Illuminate\Support\Facades\DB;
|
|
|
|
class UserController extends Controller
|
|
{
|
|
public function store(Request $request)
|
|
{
|
|
return User::create($request->all());
|
|
}
|
|
|
|
public function search(Request $request)
|
|
{
|
|
return DB::raw("SELECT * FROM users WHERE name = '" . $request->name . "'");
|
|
}
|
|
}
|
|
EOF
|
|
}
|
|
|
|
# Helper to strip ANSI codes
|
|
strip_ansi() {
|
|
sed 's/\x1b\[[0-9;]*m//g'
|
|
}
|
|
|
|
# Test 7.2.1: Django DEBUG = True
|
|
test_django_debug() {
|
|
test_case "7.2.1" "Django DEBUG = True detected"
|
|
|
|
local output
|
|
# Capture both stdout and stderr, strip ANSI codes
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/django" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Check for claims extraction - claims_extracted > 0 means the extractor found patterns
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "Django DEBUG = True not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.2: Django ALLOWED_HOSTS = ['*']
|
|
test_django_allowed_hosts() {
|
|
test_case "7.2.2" "Django ALLOWED_HOSTS = ['*'] detected"
|
|
|
|
local output
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/django" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Claims should be extracted - Django fixture has multiple security issues
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "Django ALLOWED_HOSTS wildcard not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.3: Flask app.debug = True
|
|
test_flask_debug() {
|
|
test_case "7.2.3" "Flask app.debug = True detected"
|
|
|
|
local output
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/flask" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Check claims extracted (Flask debug should produce claims)
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "Flask debug mode not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.4: Flask WTF_CSRF_ENABLED = False
|
|
test_flask_csrf() {
|
|
test_case "7.2.4" "Flask WTF_CSRF_ENABLED = False detected"
|
|
|
|
local output
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/flask" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Check claims extracted
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "Flask CSRF disabled not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.5: Spring csrf().disable()
|
|
test_spring_csrf() {
|
|
test_case "7.2.5" "Spring csrf().disable() detected"
|
|
|
|
local output
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/spring" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Check claims extracted
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "Spring CSRF disable not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.6: Spring permitAll()
|
|
test_spring_permit_all() {
|
|
test_case "7.2.6" "Spring permitAll() detected"
|
|
|
|
local output
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/spring" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Check claims extracted - Spring fixtures should produce claims
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "Spring permitAll not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.7: Express CORS wildcard with credentials
|
|
test_express_cors() {
|
|
test_case "7.2.7" "Express cors({ origin: '*', credentials: true }) detected"
|
|
|
|
local output
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/express" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Check claims extracted
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "Express CORS wildcard with credentials not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.8: Rails protect_from_forgery with: :null_session
|
|
test_rails_csrf() {
|
|
test_case "7.2.8" "Rails protect_from_forgery with: :null_session detected"
|
|
|
|
local output
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/rails" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Check claims extracted
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "Rails CSRF null_session not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.9: FastAPI CORS wildcard with credentials
|
|
test_fastapi_cors() {
|
|
test_case "7.2.9" "FastAPI allow_origins=['*'], allow_credentials=True detected"
|
|
|
|
local output
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/fastapi" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Check claims extracted
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "FastAPI CORS wildcard with credentials not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.10: Laravel APP_DEBUG=true
|
|
test_laravel_debug() {
|
|
test_case "7.2.10" "Laravel APP_DEBUG=true detected"
|
|
|
|
local output
|
|
output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/laravel" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# Check claims extracted
|
|
if echo "$output" | grep -qE 'claims_extracted=[1-9][0-9]*'; then
|
|
pass
|
|
else
|
|
fail "Laravel APP_DEBUG not detected (no claims extracted)"
|
|
echo " Output: $(echo "$output" | head -20)"
|
|
fi
|
|
}
|
|
|
|
# Test 7.2.11: Cross-framework consistency
|
|
test_cross_framework_consistency() {
|
|
test_case "7.2.11" "Same security issue detected across multiple frameworks"
|
|
|
|
# Check that all frameworks have claims extracted
|
|
local django_output flask_output spring_output
|
|
django_output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/django" --format json 2>&1 | strip_ansi || true)
|
|
flask_output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/flask" --format json 2>&1 | strip_ansi || true)
|
|
spring_output=$("$APHORIA_BIN" scan "${FIXTURES_DIR}/spring" --format json 2>&1 | strip_ansi || true)
|
|
|
|
# All should extract claims (shown in logs as claims_extracted > 0)
|
|
local django_has_claims flask_has_claims spring_has_claims
|
|
django_has_claims=$(echo "$django_output" | grep -qE 'claims_extracted=[1-9][0-9]*' && echo "yes" || echo "no")
|
|
flask_has_claims=$(echo "$flask_output" | grep -qE 'claims_extracted=[1-9][0-9]*' && echo "yes" || echo "no")
|
|
spring_has_claims=$(echo "$spring_output" | grep -qE 'claims_extracted=[1-9][0-9]*' && echo "yes" || echo "no")
|
|
|
|
if [[ "$django_has_claims" == "yes" && "$flask_has_claims" == "yes" && "$spring_has_claims" == "yes" ]]; then
|
|
pass
|
|
else
|
|
fail "All frameworks should extract claims (django=$django_has_claims, flask=$flask_has_claims, spring=$spring_has_claims)"
|
|
fi
|
|
}
|
|
|
|
# Run all tests
|
|
main() {
|
|
echo "========================================"
|
|
echo "Aphoria Framework Security UAT"
|
|
echo "========================================"
|
|
|
|
create_fixtures
|
|
|
|
echo ""
|
|
echo "Running framework security tests..."
|
|
|
|
test_django_debug
|
|
test_django_allowed_hosts
|
|
test_flask_debug
|
|
test_flask_csrf
|
|
test_spring_csrf
|
|
test_spring_permit_all
|
|
test_express_cors
|
|
test_rails_csrf
|
|
test_fastapi_cors
|
|
test_laravel_debug
|
|
test_cross_framework_consistency
|
|
|
|
echo ""
|
|
echo "========================================"
|
|
echo "Results: $PASSED/$TOTAL passed, $FAILED failed"
|
|
echo "========================================"
|
|
|
|
if [[ $FAILED -gt 0 ]]; then
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
main "$@"
|