From acae61bfcc17784225f43b726511e9924009d5cb Mon Sep 17 00:00:00 2001
From: emranemran <emran.mah@gmail.com>
Date: Sun, 7 Apr 2024 14:30:48 -0700
Subject: [PATCH] transmux: use smaller chunks when concatenating long list of
 files

When very large files are ingested, the segmented file list can be a
very long list. This list of files are passed to ffmpeg commands which
will may fail with a 'command: arg list too long' error (linux error and
not ffmpeg error). This happens because the arg length exceeds the
MAX_ARG limit which varies from kernel to kernel. To workaround this
issue, we break down the list into smaller chunks and then concat those
chunks to get the final concatenated file.
---
 video/transmux.go      | 40 +++++++++++++++++++++++++++++++++-
 video/transmux_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 1 deletion(-)

diff --git a/video/transmux.go b/video/transmux.go
index 814a1152d..ee302fb86 100644
--- a/video/transmux.go
+++ b/video/transmux.go
@@ -18,6 +18,7 @@ import (
 
 const (
 	Mp4DurationLimit = 21600 //MP4s will be generated only for first 6 hours
+	MaxArgLimit      = 250
 )
 
 func MuxTStoMP4(tsInputFile, mp4OutputFile string) ([]string, error) {
@@ -141,6 +142,29 @@ func ConcatTS(tsFileName string, segmentsList *TSegmentList, sourceMediaPlaylist
 				break
 			}
 		}
+		// If the argument list of files gets too long, linux might complain about exceeding
+		// MAX_ARG limit and ffmpeg (or any other command) using the long list will fail to run.
+		// So we split into chunked files then concat it one final time to get the final file.
+		if len(segmentFilenames) > MaxArgLimit {
+			chunks := ConcatChunkedFiles(segmentFilenames, MaxArgLimit)
+
+			var chunkFiles []string
+			for idx, chunk := range chunks {
+				concatArg := "concat:" + strings.Join(chunk, "|")
+				chunkFilename := fileBaseWithoutExt + "_" + "chunk" + strconv.Itoa(idx) + ".ts"
+				chunkFiles = append(chunkFiles, chunkFilename)
+				err := concatFiles(concatArg, chunkFilename)
+				if err != nil {
+					return totalBytes, fmt.Errorf("failed to file-concat a chunk (#%d)into a ts file: %w", idx, err)
+				}
+			}
+			if len(chunkFiles) == 0 {
+				return totalBytes, fmt.Errorf("failed to generate chunks to concat")
+			}
+			// override with the chunkFilenames instead
+			segmentFilenames = chunkFiles
+
+		}
 		concatArg := "concat:" + strings.Join(segmentFilenames, "|")
 
 		// Use file-based concatenation by reading segment files in text file
@@ -150,7 +174,6 @@ func ConcatTS(tsFileName string, segmentsList *TSegmentList, sourceMediaPlaylist
 		}
 
 		return totalBytes, nil
-
 	} else {
 		// Create a text file containing filenames of the segments
 		segmentListTxtFileName := fileBaseWithoutExt + ".txt"
@@ -255,3 +278,18 @@ func concatFiles(segmentList, outputTsFileName string) error {
 	}
 	return nil
 }
+
+// ConcatChunkedFiles splits the segmentFilenames into smaller chunks based on the maxLength value,
+// where maxLength is the maximum number of filenames per chunk.
+func ConcatChunkedFiles(filenames []string, maxLength int) [][]string {
+	var chunks [][]string
+	for maxLength > 0 && len(filenames) > 0 {
+		if len(filenames) <= maxLength {
+			chunks = append(chunks, filenames)
+			break
+		}
+		chunks = append(chunks, filenames[:maxLength])
+		filenames = filenames[maxLength:]
+	}
+	return chunks
+}
diff --git a/video/transmux_test.go b/video/transmux_test.go
index 2803e86d4..81860ab3d 100644
--- a/video/transmux_test.go
+++ b/video/transmux_test.go
@@ -5,6 +5,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"os"
 	"path/filepath"
+	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 )
@@ -213,6 +215,53 @@ func TestItConcatsStreamsOnlyUptoMP4DurationLimit(t *testing.T) {
 	require.Equal(t, int64(406268), totalBytesW)
 }
 
+func TestConcatChunkedFiles(t *testing.T) {
+	filenames := make([]string, 10)
+	for i := range filenames {
+		filenames[i] = "file" + strconv.Itoa(i+1)
+	}
+
+	testCases := []struct {
+		name       string
+		maxLength  int
+		wantChunks [][]string
+	}{
+		{
+			name:      "MaxLengthLessThanLength",
+			maxLength: 3,
+			wantChunks: [][]string{
+				{"file1", "file2", "file3"},
+				{"file4", "file5", "file6"},
+				{"file7", "file8", "file9"},
+				{"file10"},
+			},
+		},
+		{
+			name:      "MaxLengthEqualToLength",
+			maxLength: 10,
+			wantChunks: [][]string{
+				{"file1", "file2", "file3", "file4", "file5", "file6", "file7", "file8", "file9", "file10"},
+			},
+		},
+		{
+			name:      "MaxLengthGreaterThanLength",
+			maxLength: 15,
+			wantChunks: [][]string{
+				{"file1", "file2", "file3", "file4", "file5", "file6", "file7", "file8", "file9", "file10"},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			gotChunks := ConcatChunkedFiles(filenames, tc.maxLength)
+			if !reflect.DeepEqual(gotChunks, tc.wantChunks) {
+				t.Errorf("ConcatChunkedFiles(%v, %d) = %v, want %v", filenames, tc.maxLength, gotChunks, tc.wantChunks)
+			}
+		})
+	}
+}
+
 func populateRenditionSegmentList() *TRenditionList {
 	segmentFiles := []string{"../test/fixtures/seg-0.ts", "../test/fixtures/seg-1.ts", "../test/fixtures/seg-2.ts"}