v1

src/app/api/compress-image/route.ts

@@ -0,0 +1,187 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { createReadStream, createWriteStream, existsSync, mkdirSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import sharp from 'sharp';
+
+// Store compression progress in memory
+let compressionProgress = {
+  currentFile: '',
+  status: 'idle', // idle, processing, completed, error
+  progress: 0,
+  totalFiles: 0,
+  message: '',
+  results: [] as Array<{
+    originalPath: string; 
+    compressedPath: string; 
+    originalSize: number; 
+    compressedSize: number;
+    compressionRatio: number;
+  }>
+};
+
+export async function POST(request: NextRequest) {
+  try {
+    const body = await request.json();
+    const { imagePath } = body;
+
+    if (!imagePath) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Image path is required'
+        },
+        { status: 400 }
+      );
+    }
+
+    // Validate image path
+    // Handle both full paths and direct filenames
+    let filename = imagePath;
+    if (imagePath.includes('/')) {
+      filename = imagePath.split('/').pop() || imagePath;
+    }
+    
+    const fullPath = join(process.cwd(), 'public', 'observations', filename);
+    
+    if (!existsSync(fullPath)) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Image file not found',
+          details: `Looking for: ${filename}`,
+          skipped: true // Mark as skipped, not failed
+        },
+        { status: 404 }
+      );
+    }
+
+    // Start compression process
+    compressionProgress = {
+      currentFile: imagePath,
+      status: 'processing',
+      progress: 0,
+      totalFiles: 1,
+      message: 'Starting image compression...',
+      results: []
+    };
+
+    // Process the image
+    try {
+      // Create compressed directory if it doesn't exist
+      const compressedDir = join(process.cwd(), 'public', 'observations', 'compressed');
+      mkdirSync(compressedDir, { recursive: true });
+
+      // Generate compressed filename
+      const filename = imagePath.split('/').pop() || imagePath;
+      const nameWithExt = filename.split('.');
+      const extension = nameWithExt.pop();
+      const nameWithoutExt = nameWithExt.join('.');
+      const compressedFilename = `${nameWithoutExt}.jpg`; // Keep same name, ensure .jpg extension
+      const compressedPath = join('compressed', compressedFilename);
+
+      // Get original file stats
+      const stats = require('fs').statSync(fullPath);
+      const originalSize = stats.size;
+
+      // Update progress
+      compressionProgress.progress = 25;
+      compressionProgress.message = 'Resizing image to 720p...';
+
+      // Process image with sharp
+      const imageBuffer = require('fs').readFileSync(fullPath);
+      
+      // Resize and compress
+      const processedImage = await sharp(imageBuffer)
+        .resize(720, null, { 
+          fit: 'inside',
+          withoutEnlargement: true
+        })
+        .jpeg({ 
+          quality: 80,
+          progressive: true 
+        })
+        .toBuffer();
+
+      // Update progress
+      compressionProgress.progress = 75;
+      compressionProgress.message = 'Compressing image...';
+
+      // Save compressed image
+      writeFileSync(join(compressedDir, compressedFilename), processedImage);
+
+      // Get compressed file stats
+      const compressedStats = require('fs').statSync(join(compressedDir, compressedFilename));
+      const compressedSize = compressedStats.size;
+
+      // Calculate compression ratio
+      const compressionRatio = ((originalSize - compressedSize) / originalSize * 100).toFixed(1);
+
+      // Update progress to completed
+      compressionProgress.progress = 100;
+      compressionProgress.status = 'completed';
+      compressionProgress.message = 'Image compression completed successfully';
+      compressionProgress.results = [{
+        originalPath: imagePath,
+        compressedPath: `compressed/${compressedFilename}`,
+        originalSize,
+        compressedSize,
+        compressionRatio: parseFloat(compressionRatio)
+      }];
+
+      return NextResponse.json({
+        success: true,
+        message: 'Image compressed successfully',
+        data: {
+          originalPath: imagePath,
+          compressedPath: `compressed/${compressedFilename}`,
+          originalSize,
+          compressedSize,
+          compressionRatio: parseFloat(compressionRatio)
+        }
+      });
+
+    } catch (error: any) {
+      compressionProgress.status = 'error';
+      compressionProgress.message = `Compression failed: ${error.message}`;
+      
+      console.error('Compression error details:', {
+        imagePath,
+        filename,
+        fullPath,
+        error: error.message,
+        stack: error.stack
+      });
+      
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Failed to compress image',
+          message: error.message,
+          details: {
+            imagePath,
+            filename,
+            fullPath
+          }
+        },
+        { status: 500 }
+      );
+    }
+
+  } catch (error: any) {
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Invalid request format'
+      },
+      { status: 400 }
+    );
+  }
+}
+
+// Progress endpoint
+export async function GET() {
+  return NextResponse.json({
+    success: true,
+    data: compressionProgress
+  });
+}

src/app/api/data-structure/route.ts

@@ -0,0 +1,42 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+
+export async function GET() {
+  try {
+    const filePath = join(process.cwd(), 'public', 'data_structure.json');
+    
+    try {
+      const fileContent = await readFile(filePath, 'utf-8');
+      const data = JSON.parse(fileContent);
+      
+      return NextResponse.json({
+        success: true,
+        data: data
+      });
+    } catch (fileError) {
+      // If file doesn't exist, return empty arrays
+      return NextResponse.json({
+        success: true,
+        data: {
+          learningAreas: [],
+          subLearningAreas: [],
+          frameworks: [],
+          lastUpdated: null,
+          totalObservations: 0
+        }
+      });
+    }
+    
+  } catch (error: any) {
+    console.error('Error reading data structure:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to read data structure',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  }
+}

src/app/api/delete-observation/[id]/route.ts

@@ -0,0 +1,95 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+export async function DELETE(
+  request: NextRequest,
+  context: { params: Promise<{ id: string }> }
+) {
+  try {
+    const params = await context.params;
+    const id = params.id;
+    
+    if (!id) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'No ID provided'
+        },
+        { status: 400 }
+      );
+    }
+
+    const { MongoClient, ObjectId } = require('mongodb');
+    const uri = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+    const client = new MongoClient(uri);
+    await client.connect();
+    const db = client.db('beanstalk');
+    const collection = db.collection('observations');
+    
+    // Try to find the document first to determine the correct _id format
+    let document = await collection.findOne({ _id: id });
+    let filter;
+    
+    if (document) {
+      // Found with string ID
+      filter = { _id: id };
+    } else {
+      // Try with ObjectId
+      try {
+        const objectId = new ObjectId(id);
+        document = await collection.findOne({ _id: objectId });
+        if (document) {
+          filter = { _id: objectId };
+        } else {
+          await client.close();
+          return NextResponse.json(
+            {
+              success: false,
+              error: 'Observation not found'
+            },
+            { status: 404 }
+          );
+        }
+      } catch (error) {
+        await client.close();
+        return NextResponse.json(
+          {
+            success: false,
+            error: 'Invalid ID format'
+          },
+          { status: 400 }
+        );
+      }
+    }
+
+    // Delete the document
+    const result = await collection.deleteOne(filter);
+    
+    await client.close();
+    
+    if (result.deletedCount === 0) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Failed to delete observation'
+        },
+        { status: 500 }
+      );
+    }
+    
+    return NextResponse.json({
+      success: true,
+      message: 'Observation deleted successfully'
+    });
+    
+  } catch (error: any) {
+    console.error('Delete observation error:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to delete observation',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  }
+}

src/app/api/frameworks/route.ts

@@ -0,0 +1,54 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { MongoClient } from 'mongodb';
+
+const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+const DATABASE_NAME = 'beanstalk';
+
+export async function GET() {
+  let client: MongoClient | null = null;
+  
+  try {
+    client = new MongoClient(MONGODB_URI);
+    await client.connect();
+    
+    const db = client.db(DATABASE_NAME);
+    
+    // Get unique frameworks from indicators collection
+    const indicators = await db.collection('indicators').find({}).toArray();
+    
+    // Extract unique frameworks
+    const frameworks = new Set<string>();
+    indicators.forEach((indicator: any) => {
+      if (indicator.developmentArea && Array.isArray(indicator.developmentArea)) {
+        indicator.developmentArea.forEach((area: any) => {
+          if (area.framework) {
+            frameworks.add(area.framework);
+          }
+        });
+      }
+    });
+    
+    return NextResponse.json({
+      success: true,
+      data: {
+        frameworks: Array.from(frameworks).sort(),
+        count: frameworks.size
+      }
+    });
+    
+  } catch (error: any) {
+    console.error('Error fetching frameworks:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to fetch frameworks',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  } finally {
+    if (client) {
+      await client.close();
+    }
+  }
+}

src/app/api/get-images-from-s3/route.ts

@@ -0,0 +1,268 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { writeFile, mkdir } from 'fs/promises';
+import { join } from 'path';
+import { createHash } from 'crypto';
+
+interface S3Response {
+  url: string;
+  filename?: string;
+}
+
+interface ProgressCallback {
+  (progress: {
+    current: number;
+    total: number;
+    currentKey: string;
+    status: 'processing' | 'success' | 'error';
+    message?: string;
+  }): void;
+}
+
+// Store progress in memory (in production, use Redis or database)
+let downloadProgress: any = {
+  current: 0,
+  total: 0,
+  currentKey: '',
+  status: 'idle',
+  results: []
+};
+
+async function downloadImage(s3Key: string): Promise<{ url: string; filename: string }> {
+  try {
+    const apiDomain = process.env.API_DOMAIN || 'https://app.example.tech';
+    const response = await fetch(`${apiDomain}/api/one/v1/file/url?s3Key=${encodeURIComponent(s3Key)}`);
+    
+    if (!response.ok) {
+      throw new Error(`Failed to get download URL for ${s3Key}: ${response.statusText}`);
+    }
+    
+    const data: S3Response = await response.json();
+    
+    if (!data.url) {
+      throw new Error(`No download URL returned for ${s3Key}`);
+    }
+    
+    // Extract filename from S3 key
+    const filename = s3Key.split('/').pop() || s3Key;
+    
+    return { url: data.url, filename };
+  } catch (error) {
+    console.error(`Error getting download URL for ${s3Key}:`, error);
+    throw error;
+  }
+}
+
+async function saveImageFromUrl(url: string, filename: string, savePath: string): Promise<void> {
+  try {
+    // Download the image
+    const response = await fetch(url);
+    
+    if (!response.ok) {
+      throw new Error(`Failed to download image ${filename}: ${response.statusText}`);
+    }
+    
+    const arrayBuffer = await response.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+    
+    // Save to file system
+    await writeFile(savePath, buffer);
+    console.log(`Saved image: ${savePath}`);
+  } catch (error) {
+    console.error(`Error saving image ${filename}:`, error);
+    throw error;
+  }
+}
+
+function extractImageKeys(documents: any[]): string[] {
+  const imageKeys = new Set<string>();
+  
+  documents.forEach(doc => {
+    // Extract from assets array
+    if (doc.assets && Array.isArray(doc.assets)) {
+      doc.assets.forEach((asset: string) => {
+        if (asset.startsWith('akademy/')) {
+          imageKeys.add(asset);
+        }
+      });
+    }
+  });
+  
+  return Array.from(imageKeys);
+}
+
+async function processImageSequentially(
+  imageKeys: string[], 
+  observationsDir: string,
+  onProgress?: ProgressCallback
+): Promise<any[]> {
+  const results = [];
+  let successCount = 0;
+  let errorCount = 0;
+
+  for (let i = 0; i < imageKeys.length; i++) {
+    const imageKey = imageKeys[i];
+    
+    try {
+      // Update progress - starting
+      downloadProgress.current = i + 1;
+      downloadProgress.total = imageKeys.length;
+      downloadProgress.currentKey = imageKey;
+      downloadProgress.status = 'processing';
+      
+      if (onProgress) {
+        onProgress({
+          ...downloadProgress,
+          status: 'processing',
+          message: `Starting download: ${imageKey}`
+        });
+      }
+
+      console.log(`[${i + 1}/${imageKeys.length}] Processing: ${imageKey}`);
+      
+      // Get download URL from S3 API
+      const { url, filename } = await downloadImage(imageKey);
+      
+      // Create a safe filename (remove path separators but keep original name)
+      const safeFilename = filename.replace(/[^a-zA-Z0-9.-]/g, '_');
+      const finalFilename = safeFilename;
+      const savePath = join(observationsDir, finalFilename);
+      
+      // Download and save image
+      await saveImageFromUrl(url, filename, savePath);
+      
+      const result = {
+        original_key: imageKey,
+        filename: finalFilename,
+        local_path: `/observations/${finalFilename}`,
+        status: 'success'
+      };
+      
+      results.push(result);
+      successCount++;
+      
+      // Update progress - success
+      downloadProgress.status = 'success';
+      if (onProgress) {
+        onProgress({
+          ...downloadProgress,
+          status: 'success',
+          message: `Successfully saved: ${finalFilename}`
+        });
+      }
+      
+      // Small delay between downloads to avoid overwhelming the API
+      await new Promise(resolve => setTimeout(resolve, 100));
+      
+    } catch (error: any) {
+      console.error(`Failed to process ${imageKey}:`, error.message);
+      
+      const result = {
+        original_key: imageKey,
+        error: error.message,
+        status: 'error'
+      };
+      
+      results.push(result);
+      errorCount++;
+      
+      // Update progress - error
+      downloadProgress.status = 'error';
+      if (onProgress) {
+        onProgress({
+          ...downloadProgress,
+          status: 'error',
+          message: `Failed: ${error.message}`
+        });
+      }
+    }
+  }
+  
+  return results;
+}
+
+export async function POST(request: NextRequest) {
+  try {
+    // Ensure the public/observations directory exists
+    const observationsDir = join(process.cwd(), 'public', 'observations');
+    try {
+      await mkdir(observationsDir, { recursive: true });
+    } catch (error: any) {
+      if (error.code !== 'EEXIST') {
+        throw error;
+      }
+    }
+
+    // Get all documents from MongoDB to extract image keys
+    const { MongoClient } = require('mongodb');
+    const uri = process.env.MONGODB_URI || 'mongodb://localhost:27017';
+    const client = new MongoClient(uri);
+    await client.connect();
+    const db = client.db('beanstalk');
+    const collection = db.collection('observations');
+    
+    const documents = await collection.find({}).toArray();
+    await client.close();
+    
+    // Extract unique image keys from documents
+    const imageKeys = extractImageKeys(documents);
+    
+    if (imageKeys.length === 0) {
+      return NextResponse.json({
+        success: true,
+        message: 'No images found in documents',
+        images_processed: 0,
+        images_saved: 0
+      });
+    }
+    
+    console.log(`Found ${imageKeys.length} unique images to download`);
+    
+    // Reset progress
+    downloadProgress = {
+      current: 0,
+      total: imageKeys.length,
+      currentKey: '',
+      status: 'starting',
+      results: []
+    };
+
+    // Process images one by one sequentially
+    const results = await processImageSequentially(imageKeys, observationsDir);
+    
+    const successCount = results.filter(r => r.status === 'success').length;
+    const errorCount = results.filter(r => r.status === 'error').length;
+    
+    return NextResponse.json({
+      success: true,
+      message: `Completed processing ${imageKeys.length} images. Successfully saved ${successCount}, failed ${errorCount}`,
+      images_processed: imageKeys.length,
+      images_saved: successCount,
+      images_failed: errorCount,
+      results: results
+    });
+    
+  } catch (error: any) {
+    console.error('Error in get-images-from-s3:', error);
+    return NextResponse.json(
+      {
+        error: 'Failed to process images from S3',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  }
+}
+
+export async function GET(request: NextRequest) {
+  // Return current progress
+  return NextResponse.json({
+    message: 'Current download progress',
+    progress: downloadProgress,
+    usage: {
+      method: 'POST',
+      endpoint: '/api/get-images-from-s3',
+      description: 'Downloads all images found in MongoDB observations and saves them to public/observations/',
+      example: 'fetch("/api/get-images-from-s3", { method: "POST" })'
+    }
+  });
+}

src/app/api/get-observation/[id]/route.ts

@@ -0,0 +1,74 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+export async function GET(
+  request: NextRequest,
+  context: { params: Promise<{ id: string }> }
+) {
+  try {
+    const params = await context.params;
+    const id = params.id;
+    console.log('=== API CALLED WITH ID:', id);
+    console.log('Full params object:', JSON.stringify(params));
+    
+    if (!id) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'No ID provided'
+        },
+        { status: 400 }
+      );
+    }
+    
+    const { MongoClient, ObjectId } = require('mongodb');
+    const uri = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+    const client = new MongoClient(uri);
+    await client.connect();
+    const db = client.db('beanstalk');
+    const collection = db.collection('observations');
+    
+    // Try to find by string ID first
+    let document = await collection.findOne({ _id: id });
+    console.log('String ID search result:', document ? 'Found' : 'Not found');
+    
+    // If not found, try ObjectId
+    if (!document) {
+      try {
+        const objectId = new ObjectId(id);
+        document = await collection.findOne({ _id: objectId });
+        console.log('ObjectId search result:', document ? 'Found' : 'Not found');
+      } catch (error: any) {
+        console.log('Invalid ObjectId format:', error.message);
+      }
+    }
+    
+    await client.close();
+    
+    if (!document) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Observation not found',
+          searchedId: id
+        },
+        { status: 404 }
+      );
+    }
+    
+    return NextResponse.json({
+      success: true,
+      data: document
+    });
+    
+  } catch (error: any) {
+    console.error('Error fetching observation:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to fetch observation',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  }
+}

src/app/api/get-observations/route.ts

@@ -0,0 +1,73 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+export async function GET(request: NextRequest) {
+  try {
+    const { searchParams } = new URL(request.url);
+    const page = parseInt(searchParams.get('page') || '1');
+    const limit = parseInt(searchParams.get('limit') || '12');
+    const framework = searchParams.get('framework');
+    const learningArea = searchParams.get('learningArea');
+    const subLearningArea = searchParams.get('subLearningArea');
+    
+    const { MongoClient } = require('mongodb');
+    const uri = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+    const client = new MongoClient(uri);
+    await client.connect();
+    const db = client.db('beanstalk');
+    const collection = db.collection('observations');
+    
+    // Build filter query
+    let filter: any = {};
+    
+    if (framework) {
+      filter['indicators.developmentAreas.framework'] = framework.trim();
+    }
+    
+    if (learningArea) {
+      filter['indicators.developmentAreas.learningArea'] = learningArea.trim();
+    }
+    
+    if (subLearningArea) {
+      filter['indicators.developmentAreas.subLearningArea'] = subLearningArea.trim();
+    }
+    
+    // Get total count for pagination
+    const totalCount = await collection.countDocuments(filter);
+    
+    // Calculate skip value for pagination
+    const skip = (page - 1) * limit;
+    
+    // Fetch observations with pagination and sort by createdAt (newest first)
+    const documents = await collection
+      .find(filter)
+      .sort({ createdAt: -1 })
+      .skip(skip)
+      .limit(limit)
+      .toArray();
+    
+    await client.close();
+    
+    return NextResponse.json({
+      success: true,
+      data: documents,
+      pagination: {
+        currentPage: page,
+        totalPages: Math.ceil(totalCount / limit),
+        totalCount: totalCount,
+        limit: limit,
+        hasNextPage: page < Math.ceil(totalCount / limit),
+        hasPrevPage: page > 1
+      }
+    });
+    
+  } catch (error: any) {
+    console.error('Error fetching observations:', error);
+    return NextResponse.json(
+      {
+        error: 'Failed to fetch observations',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  }
+}

src/app/api/import-json/route.ts

@@ -0,0 +1,188 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { MongoClient, Db, Collection, ObjectId } from 'mongodb';
+
+// MongoDB connection
+let client: MongoClient | null = null;
+let db: Db | null = null;
+
+async function getDatabase(): Promise<Db> {
+  if (!client) {
+    const uri = process.env.MONGODB_URI || 'mongodb://localhost:27017';
+    client = new MongoClient(uri);
+    await client.connect();
+    db = client.db('beanstalk');
+  }
+  return db!;
+}
+
+function convertMongoExtendedJSON(obj: any): any {
+  if (obj === null || obj === undefined) {
+    return obj;
+  }
+  
+  if (Array.isArray(obj)) {
+    return obj.map(item => convertMongoExtendedJSON(item));
+  }
+  
+  if (typeof obj === 'object') {
+    const converted: any = {};
+    
+    for (const [key, value] of Object.entries(obj)) {
+      if (key === '$oid' && typeof value === 'string') {
+        return new ObjectId(value);
+      } else if (key === '$date' && typeof value === 'string') {
+        return new Date(value);
+      } else {
+        converted[key] = convertMongoExtendedJSON(value);
+      }
+    }
+    
+    return converted;
+  }
+  
+  return obj;
+}
+
+export async function GET(request: NextRequest) {
+  try {
+    const { searchParams } = new URL(request.url);
+    const filename = searchParams.get('file');
+
+    if (!filename) {
+      return NextResponse.json(
+        { 
+          error: 'file parameter is required',
+          message: 'Usage: /api/import-json?file=db_prod.akademyObservations.json'
+        },
+        { status: 400 }
+      );
+    }
+
+    // Read JSON file
+    const fs = require('fs');
+    const path = require('path');
+    const filePath = path.join(process.cwd(), '..', filename);
+    const fileContent = fs.readFileSync(filePath, 'utf-8');
+    const rawDocuments = JSON.parse(fileContent);
+
+    // Convert MongoDB extended JSON to proper MongoDB documents
+    const documents = rawDocuments.map((doc: any) => convertMongoExtendedJSON(doc));
+
+    // Get MongoDB collection
+    const db = await getDatabase();
+    const collection = db.collection('observations');
+
+    // Insert documents
+    const result = await collection.insertMany(documents);
+
+    return NextResponse.json({
+      success: true,
+      message: `Successfully imported ${result.insertedCount} observations`,
+      inserted_count: result.insertedCount,
+      inserted_ids: Object.values(result.insertedIds)
+    });
+
+  } catch (error: any) {
+    console.error('Import error:', error);
+    
+    if (error.code === 'ENOENT') {
+      return NextResponse.json(
+        { 
+          error: 'File not found',
+          message: `The file was not found`
+        },
+        { status: 400 }
+      );
+    }
+
+    if (error.name === 'SyntaxError') {
+      return NextResponse.json(
+        { 
+          error: 'Failed to parse JSON',
+          message: error.message
+        },
+        { status: 400 }
+      );
+    }
+
+    return NextResponse.json(
+      { 
+        error: 'Failed to import data',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  }
+}
+
+export async function POST(request: NextRequest) {
+  try {
+    const formData = await request.formData();
+    const file = formData.get('file') as File;
+
+    if (!file) {
+      return NextResponse.json(
+        { 
+          error: 'No file provided',
+          message: 'Please select a JSON file to upload'
+        },
+        { status: 400 }
+      );
+    }
+
+    // Validate file type
+    if (!file.type.includes('json')) {
+      return NextResponse.json(
+        { 
+          error: 'Invalid file type',
+          message: 'Please upload a JSON file'
+        },
+        { status: 400 }
+      );
+    }
+
+    // Read file content
+    const fileContent = await file.text();
+    const rawDocuments = JSON.parse(fileContent);
+
+    // Convert MongoDB extended JSON to proper MongoDB documents
+    const documents = rawDocuments.map((doc: any) => convertMongoExtendedJSON(doc));
+
+    // Get MongoDB collection
+    const db = await getDatabase();
+    const collection = db.collection('observations');
+
+    // Insert documents
+    const result = await collection.insertMany(documents);
+
+    return NextResponse.json({
+      success: true,
+      message: `Successfully imported ${result.insertedCount} observations from ${file.name}`,
+      inserted_count: result.insertedCount,
+      inserted_ids: Object.values(result.insertedIds),
+      filename: file.name,
+      size: `${(file.size / 1024).toFixed(2)} KB`
+    });
+
+  } catch (error: any) {
+    console.error('Import error:', error);
+    
+    if (error.name === 'SyntaxError') {
+      return NextResponse.json(
+        { 
+          error: 'Failed to parse JSON',
+          message: error.message
+        },
+        { status: 400 }
+      );
+    }
+
+    return NextResponse.json(
+      { 
+        error: 'Failed to import data',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  }
+}

src/app/api/indicators-by-filter/route.ts

@@ -0,0 +1,85 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { MongoClient } from 'mongodb';
+
+const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+const DATABASE_NAME = 'beanstalk';
+
+export async function GET(request: NextRequest) {
+  let client: MongoClient | null = null;
+  
+  try {
+    client = new MongoClient(MONGODB_URI);
+    await client.connect();
+    
+    const db = client.db(DATABASE_NAME);
+    const { searchParams } = new URL(request.url);
+    
+    const framework = searchParams.get('framework');
+    const learningArea = searchParams.get('learningArea');
+    const subLearningArea = searchParams.get('subLearningArea');
+    
+    // Build filter query
+    let filter: any = {};
+    
+    if (framework) {
+      filter['developmentArea.framework'] = framework.trim();
+    }
+    
+    if (learningArea) {
+      filter['developmentArea.learningArea'] = learningArea.trim();
+    }
+    
+    if (subLearningArea) {
+      filter['developmentArea.subLearningArea'] = subLearningArea.trim();
+    }
+    
+    // Get indicators matching the filter
+    const indicators = await db.collection('indicators')
+      .find({
+        developmentArea: {
+          $elemMatch: {
+            ...(framework && { framework: framework.trim() }),
+            ...(learningArea && { learningArea: learningArea.trim() }),
+            ...(subLearningArea && { subLearningArea: subLearningArea.trim() })
+          }
+        }
+      })
+      .toArray();
+    
+    // Extract unique indicator descriptions
+    const indicatorDescriptions = new Set<string>();
+    indicators.forEach((indicator: any) => {
+      if (indicator.indicator) {
+        indicatorDescriptions.add(indicator.indicator);
+      }
+    });
+    
+    return NextResponse.json({
+      success: true,
+      data: {
+        indicators: Array.from(indicatorDescriptions).sort(),
+        count: indicatorDescriptions.size,
+        filter: {
+          framework,
+          learningArea,
+          subLearningArea
+        }
+      }
+    });
+    
+  } catch (error: any) {
+    console.error('Error fetching indicators:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to fetch indicators',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  } finally {
+    if (client) {
+      await client.close();
+    }
+  }
+}

src/app/api/indicators/route.ts

@@ -0,0 +1,50 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { MongoClient } from 'mongodb';
+
+const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+const DATABASE_NAME = 'beanstalk';
+
+export async function GET() {
+  let client: MongoClient | null = null;
+  
+  try {
+    client = new MongoClient(MONGODB_URI);
+    await client.connect();
+    
+    const db = client.db(DATABASE_NAME);
+    
+    // Get all indicators from indicators collection
+    const indicators = await db.collection('indicators').find({}).toArray();
+    
+    // Extract unique indicator descriptions
+    const indicatorDescriptions = new Set<string>();
+    indicators.forEach((indicator: any) => {
+      if (indicator.indicator) {
+        indicatorDescriptions.add(indicator.indicator);
+      }
+    });
+    
+    return NextResponse.json({
+      success: true,
+      data: {
+        indicators: Array.from(indicatorDescriptions).sort(),
+        count: indicatorDescriptions.size
+      }
+    });
+    
+  } catch (error: any) {
+    console.error('Error fetching indicators:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to fetch indicators',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  } finally {
+    if (client) {
+      await client.close();
+    }
+  }
+}

src/app/api/learning-areas/route.ts

@@ -0,0 +1,63 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { MongoClient } from 'mongodb';
+
+const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+const DATABASE_NAME = 'beanstalk';
+
+export async function GET(request: NextRequest) {
+  let client: MongoClient | null = null;
+  
+  try {
+    const { searchParams } = new URL(request.url);
+    const framework = searchParams.get('framework');
+    
+    client = new MongoClient(MONGODB_URI);
+    await client.connect();
+    
+    const db = client.db(DATABASE_NAME);
+    
+    let query: any = {};
+    if (framework) {
+      query['developmentArea.framework'] = framework;
+    }
+    
+    // Get unique learning areas from indicators collection
+    const indicators = await db.collection('indicators').find(query).toArray();
+    
+    // Extract unique learning areas
+    const learningAreas = new Set<string>();
+    indicators.forEach((indicator: any) => {
+      if (indicator.developmentArea && Array.isArray(indicator.developmentArea)) {
+        indicator.developmentArea.forEach((area: any) => {
+          if (area.learningArea) {
+            learningAreas.add(area.learningArea);
+          }
+        });
+      }
+    });
+    
+    return NextResponse.json({
+      success: true,
+      data: {
+        learningAreas: Array.from(learningAreas).sort(),
+        count: learningAreas.size,
+        framework: framework || 'all'
+      }
+    });
+    
+  } catch (error: any) {
+    console.error('Error fetching learning areas:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to fetch learning areas',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  } finally {
+    if (client) {
+      await client.close();
+    }
+  }
+}

src/app/api/proxy-image/route.ts

@@ -0,0 +1,51 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+
+export async function GET(request: NextRequest) {
+  try {
+    const { searchParams } = new URL(request.url);
+    const path = searchParams.get('path');
+    
+    if (!path) {
+      return NextResponse.json({ error: 'No path provided' }, { status: 400 });
+    }
+    
+    // Security check - ensure the path doesn't contain directory traversal
+    if (path.includes('..') || path.includes('/') || path.includes('\\')) {
+      return NextResponse.json({ error: 'Invalid path' }, { status: 400 });
+    }
+    
+    const imagePath = join(process.cwd(), 'public', 'observations', path);
+    
+    try {
+      const imageBuffer = await readFile(imagePath);
+      
+      // Determine content type based on file extension
+      const ext = path.split('.').pop()?.toLowerCase();
+      let contentType = 'image/jpeg';
+      
+      if (ext === 'png') contentType = 'image/png';
+      else if (ext === 'gif') contentType = 'image/gif';
+      else if (ext === 'webp') contentType = 'image/webp';
+      else if (ext === 'jpg' || ext === 'jpeg') contentType = 'image/jpeg';
+      
+      return new NextResponse(imageBuffer, {
+        headers: {
+          'Content-Type': contentType,
+          'Cache-Control': 'public, max-age=31536000', // Cache for 1 year
+          'Access-Control-Allow-Origin': '*',
+          'Access-Control-Allow-Methods': 'GET',
+          'Access-Control-Allow-Headers': 'Content-Type',
+        },
+      });
+    } catch (fileError) {
+      console.error('File not found:', imagePath);
+      return NextResponse.json({ error: 'Image not found' }, { status: 404 });
+    }
+    
+  } catch (error) {
+    console.error('Proxy image error:', error);
+    return NextResponse.json({ error: 'Internal server error' }, { status: 500 });
+  }
+}

src/app/api/sub-learning-areas/route.ts

@@ -0,0 +1,89 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { MongoClient } from 'mongodb';
+
+const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+const DATABASE_NAME = 'beanstalk';
+
+export async function GET(request: NextRequest) {
+  let client: MongoClient | null = null;
+  
+  try {
+    const { searchParams } = new URL(request.url);
+    const framework = searchParams.get('framework');
+    const learningArea = searchParams.get('learningArea');
+    
+    client = new MongoClient(MONGODB_URI);
+    await client.connect();
+    
+    const db = client.db(DATABASE_NAME);
+    
+    let query: any = {};
+    if (framework && learningArea) {
+      query['developmentArea'] = {
+        $elemMatch: {
+          framework: framework,
+          learningArea: learningArea
+        }
+      };
+    } else if (framework) {
+      query['developmentArea.framework'] = framework;
+    } else if (learningArea) {
+      query['developmentArea.learningArea'] = learningArea;
+    }
+    
+    // Get indicators matching the criteria
+    const indicators = await db.collection('indicators').find(query).toArray();
+    
+    // Extract unique sub-learning areas
+    const subLearningAreas = new Set<string>();
+    indicators.forEach((indicator: any) => {
+      if (indicator.developmentArea && Array.isArray(indicator.developmentArea)) {
+        indicator.developmentArea.forEach((area: any) => {
+          if (area.subLearningArea) {
+            // Only add if it matches both framework and learning area if both are provided
+            if (framework && learningArea) {
+              if (area.framework === framework && area.learningArea === learningArea) {
+                subLearningAreas.add(area.subLearningArea);
+              }
+            } else if (framework && !learningArea) {
+              if (area.framework === framework) {
+                subLearningAreas.add(area.subLearningArea);
+              }
+            } else if (!framework && learningArea) {
+              if (area.learningArea === learningArea) {
+                subLearningAreas.add(area.subLearningArea);
+              }
+            } else {
+              subLearningAreas.add(area.subLearningArea);
+            }
+          }
+        });
+      }
+    });
+    
+    return NextResponse.json({
+      success: true,
+      data: {
+        subLearningAreas: Array.from(subLearningAreas).sort(),
+        count: subLearningAreas.size,
+        framework: framework || 'all',
+        learningArea: learningArea || 'all'
+      }
+    });
+    
+  } catch (error: any) {
+    console.error('Error fetching sub-learning areas:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to fetch sub-learning areas',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  } finally {
+    if (client) {
+      await client.close();
+    }
+  }
+}

src/app/api/update-data-structure/route.ts

@@ -0,0 +1,73 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { writeFile, mkdir } from 'fs/promises';
+import { join } from 'path';
+
+export async function POST() {
+  try {
+    const { MongoClient } = require('mongodb');
+    const uri = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+    const client = new MongoClient(uri);
+    await client.connect();
+    const db = client.db('beanstalk');
+    const collection = db.collection('observations');
+    
+    // Get all observations
+    const observations = await collection.find({}).toArray();
+    
+    // Extract unique values
+    const learningAreas = new Set<string>();
+    const subLearningAreas = new Set<string>();
+    const frameworks = new Set<string>();
+    
+    observations.forEach((obs: any) => {
+      if (obs.indicators && Array.isArray(obs.indicators)) {
+        obs.indicators.forEach((indicator: any) => {
+          if (indicator.developmentAreas && Array.isArray(indicator.developmentAreas)) {
+            indicator.developmentAreas.forEach((area: any) => {
+              if (area.learningArea) learningAreas.add(area.learningArea);
+              if (area.subLearningArea) subLearningAreas.add(area.subLearningArea);
+              if (area.framework) frameworks.add(area.framework);
+            });
+          }
+        });
+      }
+    });
+    
+    const dataStructure = {
+      learningAreas: Array.from(learningAreas).sort(),
+      subLearningAreas: Array.from(subLearningAreas).sort(),
+      frameworks: Array.from(frameworks).sort(),
+      lastUpdated: new Date().toISOString(),
+      totalObservations: observations.length
+    };
+    
+    // Save to public directory so it can be accessed by the client
+    const publicDir = join(process.cwd(), 'public');
+    const filePath = join(publicDir, 'data_structure.json');
+    
+    // Ensure public directory exists
+    await mkdir(publicDir, { recursive: true });
+    
+    // Write the file
+    await writeFile(filePath, JSON.stringify(dataStructure, null, 2), 'utf-8');
+    
+    await client.close();
+    
+    return NextResponse.json({
+      success: true,
+      message: 'Data structure updated successfully',
+      data: dataStructure
+    });
+    
+  } catch (error: any) {
+    console.error('Error updating data structure:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to update data structure',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  }
+}

src/app/api/update-observation/[id]/route.ts

@@ -0,0 +1,110 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+export async function PUT(
+  request: NextRequest,
+  context: { params: Promise<{ id: string }> }
+) {
+  try {
+    const params = await context.params;
+    const id = params.id;
+    
+    if (!id) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'No ID provided'
+        },
+        { status: 400 }
+      );
+    }
+
+    const { MongoClient, ObjectId } = require('mongodb');
+    const uri = process.env.MONGODB_URI || 'mongodb://localhost/beanstalk';
+    const client = new MongoClient(uri);
+    await client.connect();
+    const db = client.db('beanstalk');
+    const collection = db.collection('observations');
+    
+    const body = await request.json();
+    
+    // Try to find the document first to determine the correct _id format
+    let document = await collection.findOne({ _id: id });
+    let filter;
+    
+    if (document) {
+      // Found with string ID
+      filter = { _id: id };
+    } else {
+      // Try with ObjectId
+      try {
+        const objectId = new ObjectId(id);
+        document = await collection.findOne({ _id: objectId });
+        if (document) {
+          filter = { _id: objectId };
+        } else {
+          await client.close();
+          return NextResponse.json(
+            {
+              success: false,
+              error: 'Observation not found'
+            },
+            { status: 404 }
+          );
+        }
+      } catch (error) {
+        await client.close();
+        return NextResponse.json(
+          {
+            success: false,
+            error: 'Invalid ID format'
+          },
+          { status: 400 }
+        );
+      }
+    }
+    
+    // Update observation
+    const result = await collection.updateOne(
+      filter,
+      { 
+        $set: {
+          ...body,
+          updatedAt: new Date()
+        }
+      }
+    );
+    
+    if (result.matchedCount === 0) {
+      await client.close();
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Observation not found'
+        },
+        { status: 404 }
+      );
+    }
+    
+    // Fetch updated document
+    const updatedDocument = await collection.findOne(filter);
+    
+    await client.close();
+    
+    return NextResponse.json({
+      success: true,
+      data: updatedDocument,
+      message: 'Observation updated successfully'
+    });
+    
+  } catch (error: any) {
+    console.error('Error updating observation:', error);
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to update observation',
+        message: error.message
+      },
+      { status: 500 }
+    );
+  }
+}