Deployment Guide

# Variables
PROJECT_ID="neurascale"
PROJECT_NUMBER="555656387124"
POOL_NAME="github-actions-pool"
PROVIDER_NAME="github-actions-provider"
SA_NAME="github-actions"
 
# Create workload identity pool
gcloud iam workload-identity-pools create $POOL_NAME \
  --project=$PROJECT_ID \
  --location="global" \
  --display-name="GitHub Actions Pool"
 
# Create workload identity provider
gcloud iam workload-identity-pools providers create-oidc $PROVIDER_NAME \
  --project=$PROJECT_ID \
  --location="global" \
  --workload-identity-pool=$POOL_NAME \
  --display-name="GitHub Actions Provider" \
  --attribute-mapping="google.subject=assertion.sub,attribute.actor=assertion.actor,attribute.repository=assertion.repository" \
  --issuer-uri="https://token.actions.githubusercontent.com" \
  --attribute-condition="assertion.repository_owner=='identity-wael'"
 
# Create service account
gcloud iam service-accounts create $SA_NAME \
  --project=$PROJECT_ID \
  --display-name="GitHub Actions Service Account"
 
# Allow workload identity to impersonate service account
gcloud iam service-accounts add-iam-policy-binding \
  ${SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com \
  --project=$PROJECT_ID \
  --role="roles/iam.workloadIdentityUser" \
  --member="principalSet://iam.googleapis.com/projects/${PROJECT_NUMBER}/locations/global/workloadIdentityPools/${POOL_NAME}/attribute.repository/identity-wael/neurascale"
 
# Grant permissions to other projects
for ENV_PROJECT in development-neurascale staging-neurascale production-neurascale; do
  gcloud projects add-iam-policy-binding $ENV_PROJECT \
    --member="serviceAccount:${SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" \
    --role="roles/editor"
done

# .pre-commit-config.yaml highlights
repos:
  # General code quality
  - repo: https://github.com/pre-commit/pre-commit-hooks
    hooks:
      - id: trailing-whitespace
      - id: end-of-file-fixer
      - id: check-yaml
      - id: check-added-large-files
        args: ['--maxkb=1000']
      - id: detect-private-key
 
  # Security scanning
  - repo: https://github.com/gitleaks/gitleaks
    hooks:
      - id: gitleaks
 
  # Frontend formatting
  - repo: https://github.com/pre-commit/mirrors-prettier
    hooks:
      - id: prettier
        files: \.(js|jsx|ts|tsx|json|css|scss|md)$
 
  # Python formatting (neural-engine)
  - repo: local
    hooks:
      - id: black-neural-engine
        name: black
        entry: scripts/dev-tools/run-black.sh
        language: system
        types: [python]
        files: ^neural-engine/.*\.py$
 
  # Terraform formatting
  - repo: https://github.com/terraform-linters/tflint
    hooks:
      - id: terraform-fmt
        files: \.tf$

# Install pre-commit
pip install pre-commit
 
# Install hooks
pre-commit install
 
# Run manually on all files
pre-commit run --all-files
 
# Skip hooks temporarily (not recommended)
git commit --no-verify -m "Emergency fix"

northamerica-northeast1-docker.pkg.dev/
├── development-neurascale/
│   └── neural-engine-development/
│       ├── neural-engine:latest
│       ├── neural-engine:v1.2.3
│       └── mcp-server:latest
├── staging-neurascale/
│   └── neural-engine-staging/
│       ├── neural-engine:latest
│       ├── neural-engine:v1.2.3
│       └── mcp-server:latest
└── production-neurascale/
    └── neural-engine-production/
        ├── neural-engine:v1.2.3  # No latest tag in prod
        └── mcp-server:v1.2.3

# .github/workflows/deploy.yml
name: Deploy NeuraScale
 
on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]
 
env:
  REGION: northamerica-northeast1
  STATE_BUCKET: neurascale-terraform-state
 
jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python 3.12.11
        uses: actions/setup-python@v4
        with:
          python-version: '3.12.11'
      - name: Run tests
        run: |
          cd neural-engine
          pip install -r requirements-dev.txt
          pytest tests/
 
  deploy:
    needs: test
    runs-on: ubuntu-latest
    steps:
      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v1
        with:
          credentials_json: ${{ secrets.GCP_SA_KEY }}
 
      - name: Deploy to environment
        run: |
          # Logic to deploy based on branch
          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
            ENV=production
          elif [[ "${{ github.ref }}" == "refs/heads/develop" ]]; then
            ENV=staging
          else
            ENV=development
          fi
 
          # Deploy using appropriate method
          ./scripts/deploy.sh $ENV

-- Core tables structure
CREATE SCHEMA IF NOT EXISTS neural;
 
-- Device registry
CREATE TABLE neural.devices (
    device_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    device_type VARCHAR(50) NOT NULL,
    device_name VARCHAR(255) NOT NULL,
    manufacturer VARCHAR(100),
    model VARCHAR(100),
    serial_number VARCHAR(255) UNIQUE,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
 
-- Sessions table
CREATE TABLE neural.sessions (
    session_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    device_id UUID REFERENCES neural.devices(device_id),
    patient_id VARCHAR(255),  -- De-identified
    start_time TIMESTAMP NOT NULL,
    end_time TIMESTAMP,
    duration_seconds INTEGER,
    channel_count INTEGER,
    sampling_rate FLOAT,
    data_quality_score FLOAT,
    metadata JSONB,
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
 
-- Annotations
CREATE TABLE neural.annotations (
    annotation_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    session_id UUID REFERENCES neural.sessions(session_id),
    timestamp TIMESTAMP NOT NULL,
    annotation_type VARCHAR(50),
    description TEXT,
    metadata JSONB,
    created_by VARCHAR(255),
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
 
-- Indexes for performance
CREATE INDEX idx_sessions_patient ON neural.sessions(patient_id);
CREATE INDEX idx_sessions_device ON neural.sessions(device_id);
CREATE INDEX idx_sessions_time ON neural.sessions(start_time, end_time);
CREATE INDEX idx_annotations_session ON neural.annotations(session_id);

-- Dataset: {environment}_neural_analytics
 
-- Partitioned sessions table
CREATE TABLE `project.dataset.neural_sessions`
PARTITION BY DATE(session_timestamp)
CLUSTER BY patient_id, device_type, signal_type
AS SELECT
  session_id,
  session_timestamp,
  patient_id,
  device_type,
  signal_type,
  duration_seconds,
  channel_count,
  sampling_rate,
  data_quality_score,
  annotations
FROM source_data;
 
-- Aggregated metrics table (hourly partitioned)
CREATE TABLE `project.dataset.neural_metrics`
PARTITION BY TIMESTAMP_TRUNC(timestamp, HOUR)
CLUSTER BY session_id, metric_type
AS SELECT
  timestamp,
  session_id,
  metric_type,
  channel_id,
  value,
  frequency_band,
  metadata
FROM metrics_stream;
 
-- ML training dataset
CREATE TABLE `project.dataset.ml_training_data` (
  sample_id STRING,
  features ARRAY<FLOAT64>,
  label STRING,
  split STRING,  -- train/validation/test
  created_timestamp TIMESTAMP
);

# .github/workflows/deploy.yml
name: Deploy NeuraScale
 
on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]
 
jobs:
  # Test Console (Frontend)
  test-console:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v2
        with:
          version: 9
 
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '18'
          cache: 'pnpm'
          cache-dependency-path: console/pnpm-lock.yaml
 
      - name: Install dependencies
        run: |
          cd console
          pnpm install
 
      - name: Run tests
        run: |
          cd console
          pnpm test
          pnpm lint
          pnpm type-check
 
      - name: Build
        run: |
          cd console
          pnpm build
 
  # Test Neural Engine (Backend)
  test-engine:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python 3.12.11
        uses: actions/setup-python@v4
        with:
          python-version: '3.12.11'
 
      - name: Install dependencies
        run: |
          cd neural-engine
          pip install -r requirements.txt
          pip install -r requirements-dev.txt
 
      - name: Run tests
        run: |
          cd neural-engine
          pytest tests/ --cov=src --cov-report=xml
          black . --check
          flake8 .
          mypy .
 
      - name: Upload coverage
        uses: codecov/codecov-action@v3
 
  # Deploy Console to Vercel
  deploy-console:
    needs: test-console
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v2
        with:
          version: 9
 
      - name: Deploy to Vercel (Preview)
        if: github.ref != 'refs/heads/main'
        run: |
          cd console
          npx vercel --token=${{ secrets.VERCEL_TOKEN }} \
            --scope=${{ secrets.VERCEL_SCOPE }}
 
      - name: Deploy to Vercel (Production)
        if: github.ref == 'refs/heads/main'
        run: |
          cd console
          npx vercel --prod --token=${{ secrets.VERCEL_TOKEN }} \
            --scope=${{ secrets.VERCEL_SCOPE }}
 
  # Deploy Neural Engine to GCP using Workload Identity Federation
  deploy-engine-staging:
    if: github.ref == 'refs/heads/develop'
    needs: test-engine
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
      id-token: 'write'  # Required for Workload Identity Federation
    steps:
      - uses: actions/checkout@v4
 
      - name: Authenticate to Google Cloud with Workload Identity
        uses: google-github-actions/auth@v1
        with:
          workload_identity_provider: 'projects/555656387124/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
          service_account: 'github-actions@neurascale.iam.gserviceaccount.com'
 
      - name: Deploy to Cloud Run
        run: |
          cd neural-engine
          gcloud run deploy neural-engine \
            --source . \
            --platform managed \
            --region northamerica-northeast1 \
            --project staging-neurascale \
            --set-env-vars "ENV=staging"
 
  deploy-engine-production:
    if: github.ref == 'refs/heads/main'
    needs: test-engine
    runs-on: ubuntu-latest
    environment: production
    permissions:
      contents: 'read'
      id-token: 'write'  # Required for Workload Identity Federation
    steps:
      - uses: actions/checkout@v4
 
      - name: Authenticate to Google Cloud with Workload Identity
        uses: google-github-actions/auth@v1
        with:
          workload_identity_provider: 'projects/555656387124/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
          service_account: 'github-actions@neurascale.iam.gserviceaccount.com'
 
      - name: Setup Helm
        uses: azure/setup-helm@v3
        with:
          version: '3.12.0'
 
      - name: Deploy to GKE
        run: |
          gcloud container clusters get-credentials neural-engine-prod \
            --region northamerica-northeast1 \
            --project production-neurascale
 
          helm upgrade --install neural-engine \
            kubernetes/helm/neural-engine/ \
            --namespace neural-engine \
            --create-namespace \
            --values kubernetes/helm/neural-engine/values-production.yaml \
            --set image.tag=${{ github.sha }}

# Vercel Deployment
VERCEL_TOKEN: your-vercel-token
VERCEL_SCOPE: your-team-or-username
 
# GCP Deployment
GCP_SA_KEY_STAGING: staging-service-account-key
GCP_SA_KEY_PRODUCTION: production-service-account-key
 
# Firebase Admin (for API routes)
FIREBASE_ADMIN_KEY: firebase-admin-sdk-key

# List revisions
gcloud run revisions list --service=neural-engine
 
# Rollback to previous revision
gcloud run services update-traffic neural-engine \
  --to-revisions=neural-engine-00042-xyz=100
 
# Zero-downtime rollback
gcloud run services update-traffic neural-engine \
  --to-revisions=neural-engine-00042-xyz=50,neural-engine-00043-abc=50
 
# Complete rollback after verification
gcloud run services update-traffic neural-engine \
  --to-revisions=neural-engine-00042-xyz=100

# Check rollout history
kubectl rollout history deployment/neural-engine -n neural-engine
 
# Rollback to previous version
kubectl rollout undo deployment/neural-engine -n neural-engine
 
# Rollback to specific revision
kubectl rollout undo deployment/neural-engine \
  --to-revision=2 -n neural-engine
 
# Monitor rollback status
kubectl rollout status deployment/neural-engine -n neural-engine

# List releases
helm list -n neural-engine
 
# Check release history
helm history neural-engine -n neural-engine
 
# Rollback to previous release
helm rollback neural-engine -n neural-engine
 
# Rollback to specific revision
helm rollback neural-engine 2 -n neural-engine

# Application health
curl https://api.neurascale.io/health
 
# Database connectivity
curl https://api.neurascale.io/health/database
 
# External dependencies
curl https://api.neurascale.io/health/dependencies
 
# Kubernetes liveness probe
kubectl describe pod neural-engine-xxx -n neural-engine

# View Cloud Run logs
gcloud logs read "resource.type=cloud_run_revision" \
  --project=production-neurascale \
  --format="value(textPayload)"
 
# View GKE logs
kubectl logs -f deployment/neural-engine -n neural-engine
 
# Search logs in Cloud Logging
gcloud logs read 'resource.type="k8s_container"
  resource.labels.namespace_name="neural-engine"' \
  --format="value(textPayload)"

# Database backups (automated)
gcloud sql backups list --instance=neurascale-prod
 
# Manual backup
gcloud sql backups create --instance=neurascale-prod
 
# Bigtable backups
gcloud bigtable backups create backup-20240115 \
  --instance=neurascale-prod \
  --cluster=neurascale-cluster \
  --table=neural_data
 
# Application data backup
gsutil -m cp -r gs://neurascale-prod-data \
  gs://neurascale-backup/$(date +%Y%m%d)