B. Imagery processing and structure from motion (SfM)

Topic 2: Structure from Motion

From images to 3D models: Photogrammetry and Structure from Motion concepts

Outline

• aerial imagery distortions
• orthorectification process
• interior and exterior orientation
• Bundle Block Adjustment
• Ground control points (GCP)
• Image mosaic and point cloud results
• Structure from motion (SfM)

Lecture

• Lecture Slides: Photogrammetry and SfM
• Lecture Recording: Fall 2024 (NCSU Only)
• Assignment Recordign: Fall 2024

Supplemental materials

• Lectures and demos from the 2021 Geological Society of America short course:
  • “Introduction to Structure from Motion Photogrammetry”
  • OpenTopography YouTube Course
• Y. Furukawa and C. Hernández . Multi-View Stereo: A Tutorial. Foundations and TrendsR in Computer Graphics and Vision, vol. 9, no. 1-2, pp. 1–148, 2013.

Assignment

• Geoprocessing UAS imagery in Agisoft Metashape

Homework

Prepare report on generating orthomosaic and Digital Surface Model using images taken by the UAS Trimble UX5 Rover and performing the processing in Agisoft Metashape. Explain the report generated by Agisoft including the data accuracy.

Structure from Motion Example

height = 600;
width = 800;

THREE = {
  const THREE = window.THREE = await require("three@0.130.0/build/three.min.js");
  await require("three@0.130.0/examples/js/controls/OrbitControls.js").catch(() => {});
  await require("three@0.130.0/examples/js/loaders/OBJLoader.js").catch(() => {});
  await require("three@0.130.0/examples/js/loaders/GLTFLoader.js").catch(() => {});
  return THREE;
}

SimplexNoise = {
    const SimplexNoise = window.SimplexNoise = await require("simplex-noise@2.4.0/simplex-noise.js");
    return SimplexNoise;
}

// Create the renderer
renderer = {
  const renderer = new THREE.WebGLRenderer({ antialias: true });
  renderer.setSize(width, height);
  return renderer;
}

// Set up scene, camera, and light
scene = new THREE.Scene();
camera = new THREE.PerspectiveCamera(75, width / height, 0.1, 1000);
camera.position.set(0, 15, 15);

scene.add(new THREE.AmbientLight(0xffffff, 0.5));

directionalLight = new THREE.DirectionalLight(0xffffff, 1);
directionalLight.position.set(5, 5, 5).normalize();

scene.add(directionalLight);

controls = {
  const controls = new THREE.OrbitControls(camera, renderer.domElement);
  controls.enableDamping = true;
  controls.dampingFactor = 0.25;
  controls.enableZoom = true;
  return controls;
}

// Create the terrain using Simplex Noise
terrainGeometry = {
  const widthSegments = 100;
  const heightSegments = 100;
  const geometry = new THREE.PlaneGeometry(10, 10, widthSegments, heightSegments);
  
  const simplex = new SimplexNoise();
  const vertices = geometry.attributes.position.array;

  // Modify the z-coordinates of the vertices using Simplex Noise
  for (let i = 0; i < vertices.length; i += 3) {
    const x = vertices[i];
    const y = vertices[i + 1];
    vertices[i + 2] = simplex.noise2D(x * 0.3, y * 0.3) / 2.0; // Reduce amplitude
  }
  
  geometry.computeVertexNormals(); // Recompute normals after modifying vertices
  return geometry;
}

// Create the terrain mesh
terrain = {
  const material = new THREE.MeshStandardMaterial({ color: 0x228b22, wireframe: true });
  const mesh = new THREE.Mesh(terrainGeometry, material);
  mesh.rotation.x = -Math.PI / 2; // Rotate to make horizontal
  scene.add(mesh);
  return mesh;
}

// Anchor points on the terrain surface, adjusted for terrain rotation
anchorPoints = {
  const positions = terrainGeometry.attributes.position.array;
  const points = [];
  
  for (let i = 0; i < 50; i++) { // Create 50 random anchor points
    const index = Math.floor(Math.random() * (positions.length / 3)) * 3;
    const point = new THREE.Vector3(positions[index], positions[index + 1], positions[index + 2]);
    // Correct for terrain rotation
    point.applyMatrix3(terrain.matrixWorld);
    points.push(point);
  }

  const anchorMaterial = new THREE.PointsMaterial({ color: 0xff0000, size: 0.25 });
  const anchorGeometry = new THREE.BufferGeometry().setFromPoints(points);
  const anchorMesh = new THREE.Points(anchorGeometry, anchorMaterial);
  scene.add(anchorMesh);
  
  return points;
}

// Create the line material and add the lines to the scene
uavlineMaterial = {
    const uavlineMaterial = new THREE.LineDashedMaterial({
    color: 0xff0000, // Red color
    dashSize: 0.1, // Length of dashes
    gapSize: 0.1, // Length of gaps between dashes
  });
  return uavlineMaterial;
}

// UAV path and camera representation, with paths as lines
uavCameras = {
  const uavPath1 = [];
  
  
  const stepSize = 0.25;
  const rowDistance = 0.25;
  const startX = -5, endX = 5, startZ = -5, endZ = 5;
  const startY = 3;
  const endY = 3;
  let direction = 1;

  for (let z = startZ; z <= endZ; z += rowDistance) {
    if (direction === 1) {
      for (let x = startX; x <= endX; x += stepSize) {
        uavPath1.push(new THREE.Vector3(x, startY, z));
      }
    } else {
      for (let x = endX; x >= startX; x -= stepSize) {
        uavPath1.push(new THREE.Vector3(x, endY, z));
      }
    }
    direction *= -1;
  }

  const uavLineGeometry = new THREE.BufferGeometry();
  uavLineGeometry.setFromPoints(uavPath1);
  const uavLine1 = new THREE.Line(uavLineGeometry, uavlineMaterial);
  uavLine1.computeLineDistances(); // Required for dashed lines
  scene.add(uavLine1);

  // Create UAV camera mesh representations
  const cameraMaterial1 = new THREE.MeshBasicMaterial({ color: 0xff0000 });

  const cameraGeometry = new THREE.SphereGeometry(0.1, 32, 32);
  const uavCamera1 = new THREE.Mesh(cameraGeometry, cameraMaterial1);


  scene.add(uavCamera1);

  return { uavCamera1, uavPath1, uavLine1, uavLineGeometry };
}



uavFrustum1 = {
    const uavFrustum1 = new THREE.CameraHelper(camera);
    // scene.add(uavFrustum1);
    return uavFrustum1;
}



function createLine(pointA, pointB, color = 0xffffff) {
    const geometry = new THREE.BufferGeometry().setFromPoints([pointA, pointB]);
    const material = new THREE.LineBasicMaterial({ color });
    const line = new THREE.Line(geometry, material);
    scene.add(line);
    return line;
}

linesPass1 = [];

linePass1 = {
    anchorPoints.forEach(point => {
        linesPass1.push(createLine(uavCameras.uavPath1[0], point, 0x00ffff));
    });
    return linesPass1;
}
        // Create lines from the UAV cameras to anchor points


uavFOVPlane = {
     // Create a plane to represent the UAV's field of view
    const planeGeometry = new THREE.PlaneGeometry(2, 2); // Adjust size as needed
    const planeMaterial = new THREE.MeshBasicMaterial({ color: 0x00ff00, side: THREE.DoubleSide }); // Solid fill color
    const uavFOVPlane = new THREE.Mesh(planeGeometry, planeMaterial);
    scene.add(uavFOVPlane);
    return uavFOVPlane;
}

// Animation loop, moving the UAVs along their paths
{
  let uavIndex = 0;
  let lastUpdateTime = 0;
  const updateInterval = 50; // Milliseconds between updates
  
  while (true) {
    requestAnimationFrame(() => {
      const { uavCamera1, uavPath1, uavLine1, uavLineGeometry } = uavCameras;
      // Move UAV cameras along the path

            // Move the UAV cameras along their paths
            uavIndex = (uavIndex + 1) % uavPath1.length;
            uavCamera1.position.copy(uavPath1[uavIndex]);
            // uavCamera2.position.copy(uavPath2[uavIndex]);

            if (uavIndex < uavPath1.length - 1) {
                uavIndex++;
                const newPoints = uavPath1.slice(0, uavIndex + 1);
                uavLineGeometry.setFromPoints(newPoints);
                uavLine1.computeLineDistances(); // Required for dashed lines
            }

            // Update the positions of the lines to keep them connected to the moving UAV cameras
            linesPass1.forEach((line, index) => {
                const linePositions = line.geometry.attributes.position.array;
                const anchorPoint = anchorPoints[index];

                // Update the line's start point (UAV camera position)
                linePositions[0] = uavCamera1.position.x;
                linePositions[1] = uavCamera1.position.y;
                linePositions[2] = uavCamera1.position.z;

                // Update the line's endpoint (anchor point on the terrain)
                linePositions[3] = anchorPoint.x;
                linePositions[4] = anchorPoint.y;
                linePositions[5] = anchorPoint.z;

                line.geometry.attributes.position.needsUpdate = true;
            });

            // // Update the UAV's field of view plane position and orientation
            uavFOVPlane.position.set(uavCamera1.position.x, 0, uavCamera1.position.z); // Set y to 0 to place it on the terrain surface
            uavFOVPlane.rotation.x = -Math.PI / 2; // Rotate to align with the terrain surface
        

      controls.update(); // Update orbit controls
      renderer.render(scene, camera); // Render scene
    });
    await Promises.delay(updateInterval);
  }
}

{
  window.addEventListener('resize', () => {
    renderer.setSize(width, height);
    camera.aspect = width / height;
    camera.updateProjectionMatrix();
  });
}

renderer.domElement