Adding option to concatenate files before processing

setting the option 'group_first=true' in the template tag (requires django-compressor#285) will cause files of like type to be concatenated before being handed to precompilers/filters.  This will allow people to do standard code decomposition with things like less files (having a mixins file separate from page-specific files)

compressor/base.py

@@ -50,6 +50,28 @@ def split_contents(self):
         """
         raise NotImplementedError
 
+    def group_contents(self):
+        contents = []
+        groups = {}
+        for kind, value, basename, elems in self.split_contents():
+            attrs = self.parser.elem_attribs(elems[0])
+            charset = attrs.get("charset", self.charset)
+            mimetype = attrs.get("type", None)
+            if mimetype:
+                if kind == SOURCE_FILE:
+                    value = self.get_filecontent(value, charset)
+                idx = groups.get(mimetype, -1)
+                if idx >= 0:
+                    contents[idx][0] = SOURCE_HUNK
+                    contents[idx][1] += value
+                    contents[idx][3].extend(elems)
+                else:
+                    groups[mimetype] = len(contents)
+                    contents.append([kind, value, smart_unicode(basename), elems])
+            else:
+                contents.append([kind, value, basename, elems])
+        return contents
+
     def get_template_name(self, mode):
         """
         Returns the template path for the given mode.
@@ -147,14 +169,17 @@ def hunks(self, forced=False):
         bunch of precompiled and/or rendered hunks.
         """
         enabled = settings.COMPRESS_ENABLED or forced
+        group_first = getattr(self, 'opts', None) and self.opts.get('group_first', 'false').lowercase() == 'true'
+        contents = group_first and self.group_contents() or self.split_contents()
 
-        for kind, value, basename, elem in self.split_contents():
+        for kind, value, basename, elems in contents:
             precompiled = False
-            attribs = self.parser.elem_attribs(elem)
+            # If it's a grouped set, they should all have the same charset and type
+            attribs = self.parser.elem_attribs(elems[0])
             charset = attribs.get("charset", self.charset)
             options = {
                 'method': METHOD_INPUT,
-                'elem': elem,
+                'elems': elems,
                 'kind': kind,
                 'basename': basename,
             }
@@ -174,7 +199,7 @@ def hunks(self, forced=False):
                     value = self.handle_output(kind, value, forced=True, basename=basename)
                     yield smart_unicode(value, charset.lower())
                 else:
-                    yield self.parser.elem_str(elem)
+                    yield "\n".join([self.parser.elem_str(e) for e in elems])
 
     def filter_output(self, content):
         """
@@ -193,10 +218,10 @@ def filter_input(self, forced=False):
             content.append(hunk)
         return content
 
-    def precompile(self, content, kind=None, elem=None, filename=None, **kwargs):
+    def precompile(self, content, kind=None, elems=None, filename=None, **kwargs):
         if not kind:
             return False, content
-        attrs = self.parser.elem_attribs(elem)
+        attrs = self.parser.elem_attribs(elems[0])
         mimetype = attrs.get("type", None)
         if mimetype:
             command = self.all_mimetypes.get(mimetype)

compressor/css.py

@@ -21,9 +21,9 @@ def split_contents(self):
             if elem_name == 'link' and elem_attribs['rel'].lower() == 'stylesheet':
                 basename = self.get_basename(elem_attribs['href'])
                 filename = self.get_filename(basename)
-                data = (SOURCE_FILE, filename, basename, elem)
+                data = (SOURCE_FILE, filename, basename, [elem])
             elif elem_name == 'style':
-                data = (SOURCE_HUNK, self.parser.elem_content(elem), None, elem)
+                data = (SOURCE_HUNK, self.parser.elem_content(elem), None, [elem])
             if data:
                 self.split_content.append(data)
                 media = elem_attribs.get('media', None)

compressor/js.py

@@ -17,9 +17,9 @@ def split_contents(self):
             if 'src' in attribs:
                 basename = self.get_basename(attribs['src'])
                 filename = self.get_filename(basename)
-                content = (SOURCE_FILE, filename, basename, elem)
+                content = (SOURCE_FILE, filename, basename, [elem])
                 self.split_content.append(content)
             else:
                 content = self.parser.elem_content(elem)
-                self.split_content.append((SOURCE_HUNK, content, None, elem))
+                self.split_content.append((SOURCE_HUNK, content, None, [elem]))
         return self.split_content

compressor/tests/media/css/one.less

@@ -0,0 +1 @@
+body { background:#990; }

compressor/tests/media/css/two.less

@@ -0,0 +1 @@
+body { color:#fff; }

compressor/tests/media/js/two.coffee

@@ -0,0 +1 @@
+# this is a comment.

compressor/tests/test_base.py

@@ -46,7 +46,7 @@ def test_css_split(self):
             (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'two.css'), u'css/two.css', u'<link rel="stylesheet" href="/media/css/two.css" type="text/css" />'),
         ]
         split = self.css_node.split_contents()
-        split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3])) for x in split]
+        split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3][0])) for x in split]
         self.assertEqual(out, split)
 
     def test_css_hunks(self):
@@ -83,7 +83,7 @@ def test_js_split(self):
             (SOURCE_HUNK, u'obj.value = "value";', None, '<script type="text/javascript">obj.value = "value";</script>'),
         ]
         split = self.js_node.split_contents()
-        split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3])) for x in split]
+        split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3][0])) for x in split]
         self.assertEqual(out, split)
 
     def test_js_hunks(self):
@@ -135,6 +135,60 @@ def test_custom_output_dir(self):
             settings.COMPRESS_OUTPUT_DIR = old_output_dir
 
 
+def make_elems_str(parser, elems):
+    return "".join([parser.elem_str(x) for x in elems])
+
+
+class CompressorGroupFirstTestCase(TestCase):
+    def setUp(self):
+        settings.COMPRESS_ENABLED = True
+        settings.COMPRESS_PRECOMPILERS = {}
+        self.css = """\
+<link rel="stylesheet" href="/media/css/one.css" type="text/css" />
+<style type="text/css">p { border:5px solid green;}</style>
+<link rel="stylesheet" href="/media/css/one.less" type="text/less" />
+<link rel="stylesheet" href="/media/css/two.less" type="text/less" />"""
+        self.css_node = CssCompressor(self.css)
+        self.css_node.opts = {'group_first': 'true'}
+
+        self.js = """\
+<script src="/media/js/one.js" type="text/javascript"></script>
+<script type="text/javascript">obj.value = "value";</script>
+<script src="/media/js/one.coffee" type="text/coffeescript"></script>
+<script src="/media/js/two.coffee" type="text/coffeescript"></script>"""
+        self.js_node = JsCompressor(self.js)
+
+    def test_css_group(self):
+        out = [
+            [SOURCE_HUNK,
+             u'body { background:#990; }p { border:5px solid green;}',
+             u'css/one.css',
+             u'<link rel="stylesheet" href="/media/css/one.css" type="text/css" /><style type="text/css">p { border:5px solid green;}</style>'],
+            [SOURCE_HUNK,
+             u'body { background:#990; }body { color:#fff; }',
+             u'css/one.less',
+             u'<link rel="stylesheet" href="/media/css/one.less" type="text/less" /><link rel="stylesheet" href="/media/css/two.less" type="text/less" />'],
+        ]
+        split = self.css_node.group_contents()
+        split = [[x[0], x[1], x[2], make_elems_str(self.css_node.parser, x[3])] for x in split]
+        self.assertEqual(out, split)
+
+    def test_js_group(self):
+        out = [
+            [SOURCE_HUNK,
+             u'obj = {};obj.value = "value";',
+             u'js/one.js',
+             '<script src="/media/js/one.js" type="text/javascript"></script><script type="text/javascript">obj.value = "value";</script>'],
+            [SOURCE_HUNK,
+             u'# this is a comment.\n# this is a comment.',
+             u'js/one.coffee',
+             '<script src="/media/js/one.coffee" type="text/coffeescript"></script><script src="/media/js/two.coffee" type="text/coffeescript"></script>'],
+        ]
+        split = self.js_node.group_contents()
+        split = [[x[0], x[1], x[2], make_elems_str(self.js_node.parser, x[3])] for x in split]
+        self.assertEqual(out, split)
+
+
 class CssMediaTestCase(TestCase):
     def setUp(self):
         self.css = """\

compressor/tests/test_parsers.py

@@ -59,7 +59,7 @@ def test_css_split(self):
             (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css', u'two.css'), u'css/two.css', u'<link href="/media/css/two.css" rel="stylesheet" type="text/css">'),
         ]
         split = self.css_node.split_contents()
-        split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3])) for x in split]
+        split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3][0])) for x in split]
         self.assertEqual(out, split)
 
     def test_js_split(self):
@@ -68,7 +68,7 @@ def test_js_split(self):
             (SOURCE_HUNK, u'obj.value = "value";', None, u'<script type="text/javascript">obj.value = "value";</script>'),
         ]
         split = self.js_node.split_contents()
-        split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3])) for x in split]
+        split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3][0])) for x in split]
         self.assertEqual(out, split)
 
 Html5LibParserTests = skipIf(