🗄️ Firestore Setup - Advanced

Introduction

Advanced Firestore patterns enable handling large-scale data, complex transactions, and sophisticated query patterns required for enterprise applications.

Key Learning Outcomes

By the end of this lesson, you'll understand:

• Sharding strategies for hot collections

• Distributed counter patterns

• Complex transaction handling

• Compound query optimization

• Data denormalization at scale

• Bulk operations and migrations

• Advanced security rules

Collection Sharding

Distributed Counter Pattern

const SHARD_COLLECTION_SIZE = 10;

async function incrementCounter(counterId) {
  const shardId = Math.floor(Math.random() * SHARD_COLLECTION_SIZE);
  const shardRef = doc(db, 'counters', counterId, 'shards', String(shardId));

  await updateDoc(shardRef, {
    value: increment(1)
  });
}

async function getCounter(counterId) {
  const snapshot = await getDocs(
    collection(db, 'counters', counterId, 'shards')
  );

  let count = 0;
  snapshot.forEach(doc => {
    count += doc.data().value || 0;
  });

  return count;
}

Sharded Collections for Writes

// For collections receiving > 1000 writes/sec
async function addDocument(data) {
  const shardId = Math.floor(Math.random() * 20);
  const shardRef = collection(db, 'users_shard_' + shardId);

  await addDoc(shardRef, data);
}

async function queryShardedCollection(whereCondition) {
  const results = [];
  const shardCount = 20;

  const promises = [];
  for (let i = 0; i < shardCount; i++) {
    const q = query(
      collection(db, 'users_shard_' + i),
      whereCondition
    );
    promises.push(getDocs(q));
  }

  const snapshots = await Promise.all(promises);
  snapshots.forEach(snapshot => {
    snapshot.forEach(doc => results.push({id: doc.id, ...doc.data()}));
  });

  return results;
}

Advanced Transactions

Multi-Document Transactions with Rollback

async function transferPoints(fromUserId, toUserId, points) {
  const result = await runTransaction(db, async (transaction) => {
    const fromRef = doc(db, 'users', fromUserId);
    const toRef = doc(db, 'users', toUserId);

    const fromSnap = await transaction.get(fromRef);
    const toSnap = await transaction.get(toRef);

    if (!fromSnap.exists() || !toSnap.exists()) {
      throw new Error('User not found');
    }

    const fromPoints = fromSnap.data().points || 0;
    if (fromPoints < points) {
      throw new Error('Insufficient points');
    }

    transaction.update(fromRef, {
      points: fromPoints - points,
      lastTransfer: serverTimestamp()
    });

    transaction.update(toRef, {
      points: (toSnap.data().points || 0) + points,
      lastReceived: serverTimestamp()
    });

    // Create transaction record
    const txnRef = collection(db, 'transactions');
    transaction.set(doc(txnRef), {
      from: fromUserId,
      to: toUserId,
      points,
      timestamp: serverTimestamp()
    });

    return {success: true, points};
  });

  return result;
}

Batch Operations at Scale

async function bulkUpdateUsers(userIds, updateData) {
  const batchSize = 500; // Firestore limit is 500 per batch

  for (let i = 0; i < userIds.length; i += batchSize) {
    const batch = writeBatch(db);
    const chunk = userIds.slice(i, i + batchSize);

    chunk.forEach(userId => {
      const userRef = doc(db, 'users', userId);
      batch.update(userRef, updateData);
    });

    await batch.commit();
    console.log(`Committed batch ${Math.floor(i / batchSize) + 1}`);
  }
}

// Bulk import data
async function bulkImportDocuments(collectionName, documents) {
  const batchSize = 500;

  for (let i = 0; i < documents.length; i += batchSize) {
    const batch = writeBatch(db);
    const chunk = documents.slice(i, i + batchSize);

    chunk.forEach((doc, idx) => {
      const ref = doc.id
        ? doc(db, collectionName, doc.id)
        : doc(db, collectionName);

      batch.set(ref, doc.data || doc);
    });

    await batch.commit();
  }
}

Complex Query Optimization

// Composite index for complex queries
async function findActiveUsers(country, subscription) {
  // Requires composite index on (country, subscription, createdAt)
  const q = query(
    collection(db, 'users'),
    where('country', '==', country),
    where('subscription', '==', subscription),
    where('isActive', '==', true),
    orderBy('createdAt', 'desc'),
    limit(100)
  );

  const snapshot = await getDocs(q);
  return snapshot.docs.map(doc => ({id: doc.id, ...doc.data()}));
}

// Alternative: Denormalization approach
async function findActiveUsersDenormalized(country, subscription) {
  // Single collection with composite field
  const q = query(
    collection(db, 'users_active'),
    where('country_subscription', '==', `${country}:${subscription}`),
    orderBy('createdAt', 'desc'),
    limit(100)
  );

  return getDocs(q);
}

Data Denormalization Strategies

class DenormalizationManager {
  async createUser(userData) {
    const userRef = doc(collection(db, 'users'));

    await setDoc(userRef, {
      id: userRef.id,
      ...userData,
      // Denormalized data for fast queries
      country_subscription: `${userData.country}:${userData.subscription}`,
      active_badge: userData.subscription === 'premium' && userData.isActive,
      lastUpdated: serverTimestamp()
    });

    return userRef.id;
  }

  async updateUserRole(userId, newRole) {
    const userRef = doc(db, 'users', userId);

    // Update user document
    await updateDoc(userRef, { role: newRole });

    // Also update denormalized copies in other collections
    const postsSnapshot = await getDocs(
      query(collection(db, 'posts'), where('authorId', '==', userId))
    );

    const batch = writeBatch(db);
    postsSnapshot.forEach(postDoc => {
      batch.update(postDoc.ref, { authorRole: newRole });
    });

    await batch.commit();
  }
}

Advanced Security Rules

firestore.rules:

rules_version = '2';
service cloud.firestore {
  match /databases/{database}/documents {
    // Function: Check if user owns document
    function isOwner(userId) {
      return request.auth.uid == resource.data.owner;
    }

    // Function: Check if user has role
    function hasRole(role) {
      return get(/databases/$(database)/documents/users/$(request.auth.uid)).data.role == role;
    }

    // Admin access to all data
    match /{document=**} {
      allow read, write: if hasRole('admin');
    }

    // User can only access own data
    match /users/{userId} {
      allow read, write: if isOwner(userId);
      allow create: if request.auth.uid == userId;
    }

    // Collaborative collections with role-based access
    match /workspaces/{workspaceId} {
      allow read: if request.auth.uid in resource.data.members;
      allow write: if isOwner(request.auth.uid) && hasRole('workspace_owner');
      
      match /documents/{documentId} {
        allow read: if request.auth.uid in get(/databases/$(database)/documents/workspaces/$(workspaceId)).data.members;
        allow write: if get(/databases/$(database)/documents/workspaces/$(workspaceId)).data.owner == request.auth.uid;
      }
    }
  }
}

Key Takeaways

• **Sharding** distributes hot collections for write throughput

• **Transactions** ensure consistency across multiple documents

• **Batch operations** efficiently process many documents

• **Denormalization** trades storage for query performance

• **Composite indexes** enable complex multi-field queries

• **Advanced rules** provide fine-grained access control

Next Steps

Learn about Firestore migration patterns from other databases, or explore integration with BigQuery for analytics.