How can I implement a custom compressed JSON field in Django to save disk space?

I'd like to help you implement a compressed JSON field in Django similar to what Elger Jonker described. Let me provide a practical implementation based on the concept.

Compressed JSON Field for Django

Here's how you can create a custom Django field that transparently compresses JSON data:

PYTHONCopied
import gzip
import json
import base64
from django.db import models
from django.db.models import JSONField


class CompressedJSONField(JSONField):
    """
    A JSONField that transparently compresses its contents using gzip.
    
    This field stores JSON data in a compressed format to save disk space
    while maintaining compatibility with Django's JSONField interface.
    """
    
    def __init__(self, *args, **kwargs):
        self.compression_level = kwargs.pop('compression_level', 4)
        super().__init__(*args, **kwargs)
    
    def from_db_value(self, value, expression, connection):
        if value is None:
            return value
            
        # Check if the value is compressed (starts with gzip magic bytes)
        if isinstance(value, str) and value.startswith('gz:'):
            # Decompress the value
            try:
                compressed_data = base64.b64decode(value[3:])
                decompressed_data = gzip.decompress(compressed_data).decode('utf-8')
                return json.loads(decompressed_data)
            except Exception as e:
                raise ValueError(f"Error decompressing JSON data: {e}")
        
        # Handle uncompressed values (for backward compatibility)
        return super().from_db_value(value, expression, connection)
    
    def get_prep_value(self, value):
        if value is None:
            return value
            
        # Convert to JSON string
        json_string = json.dumps(value)
        
        # Compress the JSON string
        compressed_data = gzip.compress(json_string.encode('utf-8'), 
                                        compresslevel=self.compression_level)
        
        # Encode as base64 for safe storage
        encoded_data = base64.b64encode(compressed_data).decode('utf-8')
        
        return f"gz:{encoded_data}"

Usage Example

Here's how you would use this field in your Django model:

PYTHONCopied
from django.db import models
from .fields import CompressedJSONField

class MyModel(models.Model):
    # Regular JSON field
    regular_data = models.JSONField(null=True, blank=True)
    
    # Compressed JSON field
    compressed_data = CompressedJSONField(null=True, blank=True)
    
    # You can also customize the compression level
    highly_compressed_data = CompressedJSONField(
        null=True, 
        blank=True,
        compression_level=9
    )

Testing the Implementation

Here's a simple test case to verify the field works correctly:

PYTHONCopied
from django.test import TestCase
from .models import MyModel

class CompressedJSONFieldTest(TestCase):
    def test_compression_and_retrieval(self):
        # Create a large JSON object
        large_data = {
            "items": [{"id": i, "value": f"test value {i}" * 100} for i in range(1000)]
        }
        
        # Save to database
        obj = MyModel.objects.create(
            regular_data=large_data,
            compressed_data=large_data
        )
        
        # Retrieve from database
        retrieved = MyModel.objects.get(id=obj.id)
        
        # Verify data integrity
        self.assertEqual(retrieved.regular_data, large_data)
        self.assertEqual(retrieved.compressed_data, large_data)
        
    def test_backward_compatibility(self):
        # Test with existing uncompressed data
        test_data = {"test": "value"}
        
        # Directly insert uncompressed JSON
        obj = MyModel.objects.create(compressed_data=test_data)
        obj.save()
        
        # Retrieve and verify
        retrieved = MyModel.objects.get(id=obj.id)
        self.assertEqual(retrieved.compressed_data, test_data)

This implementation provides transparent compression of JSON data while maintaining compatibility with Django's ORM. It's particularly useful for storing large JSON blobs that don't need to be queried directly but need to be retrieved quickly.