add support for gzip and deflate in http server and cleanup of (de)co…

…mpress functions

applications/mp4box/fileimport.c

@@ -1846,7 +1846,7 @@ GF_Err import_file(GF_ISOFile *dest, char *inName, u32 import_flags, GF_Fraction
 			e = gf_file_load_data(rvc_config, (u8 **) &data, &size);
 			GOTO_EXIT("loading RVC config file")
 
-			gf_gz_compress_payload(&data, size, &size);
+			gf_gz_compress_payload_ex (&data, size, &size, 0, GF_FALSE, NULL, GF_TRUE);
 			e |= gf_isom_set_rvc_config(dest, track, 1, 0, "application/rvc-config+xml+gz", data, size);
 			gf_free(data);
 			GOTO_EXIT("compressing and assigning RVC config")

share/doc/man/gpac-filters.1

@@ -7048,7 +7048,11 @@ When enabled, a GET on a directory will return a simple HTML listing of the cont
 .br
 
 .br
-Custom headers can be specified using .I hdrs, they apply to all requests.
+Custom headers can be specified using .I hdrs, they apply to all requests. For more advanced control on requests, use a javascript binding (see .I js and howtos).
+.br
+
+.br
+Text files are compressed using gzip or deflate if the client accepts these encodings, unless .I no_z is set.
 .br
 
 .br
@@ -7344,7 +7348,9 @@ ka (bool, default: true):      keep input alive if failure in push mode
 .br
 hdrs (strl):                   additional HTTP headers to inject, even values are names, odd values are values 
 .br
-js (str):                      jaavscript logic for server
+js (str):                      javascript logic for server
+.br
+zmax (uint, default: 50000):   maximum uncompressed size allowed for gzip or deflate compression for text files (only enabled if client indicates it), 0 will disable compression
 .br
 
 .br
@@ -8619,50 +8625,6 @@ f (strl):                      bitstream filters name - see filter help
 * (str):                       any possible options defined for AVBitstreamFilter and sub-classes. See gpac -hx ffbsf and gpac -hx ffbsf:*
 .br
 
-.br
-.SH ffbsf
-.LP
-.br
-Description: FFMPEG BitStream filter
-.br
-Version: Lavu58.9.100
-.br
-
-.br
-This filter provides bitstream filters (BSF) for compressed audio and video formats.
-.br
-See FFMPEG documentation (https://ffmpeg.org/documentation.html) for more details
-.br
-To list all supported bitstream filters for your GPAC build, use gpac -h ffbsf:*.
-.br
-
-.br
-Several BSF may be specified in .I f for different coding types. BSF not matching the coding type are silently ignored.
-.br
-When no BSF matches the input coding type, or when .I f is empty, the filter acts as a passthrough filter.
-.br
-
-.br
-Options are specified after the desired filters:
-.br
-- ffbsf:f=h264_metadata:video_full_range_flag=0
-.br
-- ffbsf:f=h264_metadata,av1_metadata:video_full_range_flag=0:color_range=tv
-.br
-
-.br
-Note: Using BSFs on some media types (e.g. avc, hevc) may trigger creation of a reframer filter (e.g. rfnalu)
-.br
-
-.br
-.SH Options (expert):
-.LP
-.br
-f (strl):                      bitstream filters name - see filter help
-.br
-* (str):                       any possible options defined for AVBitstreamFilter and sub-classes. See gpac -hx ffbsf and gpac -hx ffbsf:*
-.br
-
 .br
 .SH jsf
 .LP

src/filters/in_rtp_stream.c

@@ -502,7 +502,7 @@ GF_RTPInStream *rtpin_stream_new(GF_RTPIn *rtp, GF_SDPMedia *media, GF_SDPInfo *
 				gf_free(rvc_data);
 				return NULL;
 #else
-				gf_gz_decompress_payload(rvc_data, rvc_size, &tmp->depacketizer->sl_map.rvc_config, &tmp->depacketizer->sl_map.rvc_config_size);
+				gf_gz_decompress_payload_ex(rvc_data, rvc_size, &tmp->depacketizer->sl_map.rvc_config, &tmp->depacketizer->sl_map.rvc_config_size, GF_TRUE);
 				gf_free(rvc_data);
 #endif
 			} else {

src/filters/out_http.c

@@ -108,7 +108,7 @@ typedef struct
 #endif
 	GF_PropStringList rdirs;
 	Bool close, hold, quit, post, dlist, ice, reopen, blockio;
-	u32 port, block_size, maxc, maxp, timeout, hmode, sutc, cors, max_client_errors, max_async_buf, ka;
+	u32 port, block_size, maxc, maxp, timeout, hmode, sutc, cors, max_client_errors, max_async_buf, ka, zmax;
 	s32 max_cache_segs;
 	GF_PropStringList hdrs;
 
@@ -270,6 +270,8 @@ struct __httpout_session {
 
 	HTTP_DIRInfo *dir_desc;
 
+	u8 *comp_data;
+
 #ifdef GPAC_HAS_QJS
 	JSValue obj;
 #endif
@@ -469,6 +471,10 @@ static void httpout_close_session(GF_HTTPOutSession *sess, GF_Err code)
 	gf_dm_sess_del(sess->http_sess);
 	sess->http_sess = NULL;
 	sess->socket = NULL;
+	if (sess->comp_data) {
+		gf_free(sess->comp_data);
+		sess->comp_data = NULL;
+	}
 	sess->done = 1;
 	sess->flush_close = 0;
 	if (sess->cbk_close)
@@ -1358,6 +1364,10 @@ static void httpout_sess_io(void *usr_cbk, GF_NETIO_Parameter *parameter)
 	sess->nb_ranges = 0;
 	sess->do_log = httpout_do_log(sess, parameter->reply);
 	sess->dir_desc = NULL;
+	if (sess->comp_data) {
+		gf_free(sess->comp_data);
+		sess->comp_data=NULL;
+	}
 
 	if (!sess->async_pending && sess->ctx->on_request) {
 		sess->async_pending = 1;
@@ -1916,6 +1926,14 @@ static void httpout_sess_io(void *usr_cbk, GF_NETIO_Parameter *parameter)
 		sess->reply_code = 200;
 	}
 
+	u32 content_encoding=0;
+	const char *encode = (char *) gf_dm_sess_get_header(sess->http_sess, "Accept-Encoding");
+	if (encode) {
+		if (strstr(encode, "gzip")) content_encoding=1;
+		else if (strstr(encode, "deflate")) content_encoding=2;
+	}
+
+
 	//copy range for logs before resetting session headers
 	if (sess->do_log && range) {
 		strncpy(szRange, range, 199);
@@ -1981,6 +1999,28 @@ static void httpout_sess_io(void *usr_cbk, GF_NETIO_Parameter *parameter)
 			sess->nb_ranges = 0;
 			gf_dm_sess_set_header(sess->http_sess, "Cache-Control", "no-cache, no-store");
 		}
+		//setup compression if asked, only if no chunk transfer and no ranges
+		if (!is_head && !sess->use_chunk_transfer && !sess->nb_ranges
+			&& !sess->file_in_progress && content_encoding
+			&& (sess->file_size<=sess->ctx->zmax)
+		) {
+			u8 *data;
+			u32 osize;
+			if (gf_file_load_data_filep(sess->resource, &data, &osize)==GF_OK) {
+				u32 comp_size=0;
+				//only compress text files
+				if (gf_utf8_is_legal(data, osize)
+					&& (gf_gz_compress_payload_ex((u8 **) &data, osize, &comp_size, 0, GF_FALSE, NULL, (content_encoding==1) ? GF_TRUE : GF_FALSE) == GF_OK)
+				) {
+					sess->comp_data = data;
+					data = NULL;
+					sess->file_size = comp_size;
+					sess->bytes_in_req = comp_size;
+					gf_dm_sess_set_header(sess->http_sess, "Content-Encoding", (content_encoding==1) ? "gzip" : "deflate");
+				}
+				if (data) gf_free(data);
+			}
+		}
 		//only put content length if not using chunk transfer - bytes_in_req may be > 0 if we have a byte range on a chunk-transfer session
 		if (sess->bytes_in_req && !sess->use_chunk_transfer) {
 			sprintf(szFmt, LLU, sess->bytes_in_req);
@@ -3536,7 +3576,11 @@ static void httpout_process_session(GF_Filter *filter, GF_HTTPOutCtx *ctx, GF_HT
 		if (to_read > (u64) sess->ctx->block_size)
 			to_read = (u64) sess->ctx->block_size;
 
-		if (sess->resource) {
+		if (sess->comp_data) {
+			memcpy(sess->buffer, sess->comp_data+sess->file_pos, to_read);
+			read = to_read;
+		}
+		else if (sess->resource) {
 			read = (u32) gf_fread(sess->buffer, (u32) to_read, sess->resource);
 			//may happen when file writing is in progress
 			if (!read) {
@@ -3614,6 +3658,9 @@ static void httpout_process_session(GF_Filter *filter, GF_HTTPOutCtx *ctx, GF_HT
 		if (sess->resource) gf_fclose(sess->resource);
 		sess->resource = NULL;
 
+		if (sess->comp_data) gf_free(sess->comp_data);
+		sess->comp_data = NULL;
+
 		if (sess->nb_bytes) {
 			GF_LOG(GF_LOG_INFO, GF_LOG_HTTP, ("[HTTPOut] Done sending %s to %s ("LLU"/"LLU" bytes)\n", sess->path, sess->peer_address, sess->nb_bytes, sess->bytes_in_req));
 		}
@@ -4957,8 +5004,9 @@ static const GF_FilterArgs HTTPOutArgs[] =
 	{ OFFS(ka), "keep input alive if failure in push mode", GF_PROP_BOOL, "true", NULL, GF_FS_ARG_HINT_EXPERT},
 	{ OFFS(hdrs), "additional HTTP headers to inject, even values are names, odd values are values ", GF_PROP_STRING_LIST, NULL, NULL, GF_FS_ARG_HINT_ADVANCED},
 #ifdef GPAC_HAS_QJS
-	{ OFFS(js), "jaavscript logic for server", GF_PROP_STRING, NULL, NULL, GF_FS_ARG_HINT_EXPERT},
+	{ OFFS(js), "javascript logic for server", GF_PROP_STRING, NULL, NULL, GF_FS_ARG_HINT_EXPERT},
 #endif
+	{ OFFS(zmax), "maximum uncompressed size allowed for gzip or deflate compression for text files (only enabled if client indicates it), 0 will disable compression", GF_PROP_UINT, "50000", NULL, GF_FS_ARG_HINT_EXPERT},
 	{0}
 };
 
@@ -4997,7 +5045,9 @@ GF_FilterRegister HTTPOutRegister = {
 		"When disabled, a GET on a directory will fail.\n"
 		"When enabled, a GET on a directory will return a simple HTML listing of the content inspired from Apache.\n"
 		"  \n"
-		"Custom headers can be specified using [-hdrs](), they apply to all requests.\n"
+		"Custom headers can be specified using [-hdrs](), they apply to all requests. For more advanced control on requests, use a javascript binding (see [-js]() and howtos).\n"
+		"  \n"
+		"Text files are compressed using gzip or deflate if the client accepts these encodings, unless [-no_z]() is set.\n"
 		"  \n"
 		"# Simple HTTP server\n"
 		"In this mode, the filter does not need any input connection and exposes all files in the directories given by [-rdirs]().\n"

src/filters/out_route.c

@@ -1239,7 +1239,7 @@ static GF_Err routeout_check_service_updates(GF_ROUTEOutCtx *ctx, ROUTEService *
 	if (serv->stsid_bundle) gf_free(serv->stsid_bundle);
 	serv->stsid_bundle = (u8 *) payload_text;
 	serv->stsid_bundle_size = 1 + (u32) strlen(payload_text);
-	gf_gz_compress_payload(&serv->stsid_bundle, serv->stsid_bundle_size, &serv->stsid_bundle_size);
+	gf_gz_compress_payload_ex(&serv->stsid_bundle, serv->stsid_bundle_size, &serv->stsid_bundle_size, 0, GF_FALSE, NULL, GF_TRUE);
 
 	serv->stsid_bundle_toi = 0x80000000; //compressed
 	if (manifest_updated) serv->stsid_bundle_toi |= (1<<18);
@@ -1930,7 +1930,7 @@ static void routeout_send_lls(GF_ROUTEOutCtx *ctx)
 		comp_size = 2*len;
 		payload = gf_malloc(sizeof(char)*(comp_size+4));
 		pay_start = payload + 4;
-		gf_gz_compress_payload_ex((u8 **) &payload_text, len, &comp_size, 0, GF_FALSE, &pay_start, GF_FALSE);
+		gf_gz_compress_payload_ex((u8 **) &payload_text, len, &comp_size, 0, GF_FALSE, &pay_start, GF_TRUE);
 		gf_free(payload_text);
 		payload_text = NULL;
 
@@ -2022,7 +2022,7 @@ static void routeout_send_lls(GF_ROUTEOutCtx *ctx)
 		comp_size = 2*len;
 		payload = gf_malloc(sizeof(char)*(comp_size+4));
 		pay_start = payload + 4;
-		gf_gz_compress_payload_ex((u8 **) &payload_text, len, &comp_size, 0, GF_FALSE, &pay_start, GF_FALSE);
+		gf_gz_compress_payload_ex((u8 **) &payload_text, len, &comp_size, 0, GF_FALSE, &pay_start, GF_TRUE);
 		gf_free(payload_text);
 		payload_text = NULL;
 

src/isomedia/box_funcs.c

@@ -196,7 +196,7 @@ GF_Err gf_isom_box_parse_ex(GF_Box **outBox, GF_BitStream *bs, u32 parent_type,
 
 				compressed_size = (u32) (size - 8 - extra_bytes);
 				gf_bs_read_data(bs, compb, compressed_size);
-				e = gf_gz_decompress_payload_ex(compb, compressed_size, &uncomp_data, &osize, (do_uncompress==2) ? GF_TRUE : GF_FALSE);
+				e = gf_gz_decompress_payload_ex(compb, compressed_size, &uncomp_data, &osize, GF_FALSE);
 				if (e) {
 					gf_free(compb);
 					GF_LOG(GF_LOG_ERROR, GF_LOG_CONTAINER, ("[iso file] Failed to uncompress payload for box type %s (0x%08X)\n", gf_4cc_to_str(otype), otype));

src/isomedia/isom_store.c

@@ -385,7 +385,7 @@ GF_Err gf_isom_write_compressed_box(GF_ISOFile *mov, GF_Box *root_box, u32 repl_
 		*box_csize = (u32) root_box->size;
 
 	gf_bs_get_content(comp_bs, &box_data, &box_size);
-	gf_gz_compress_payload_ex(&box_data, box_size, &comp_size, use_cmov ? 0 : 8, GF_FALSE, NULL, use_cmov);
+	gf_gz_compress_payload_ex(&box_data, box_size, &comp_size, use_cmov ? 0 : 8, GF_FALSE, NULL, GF_FALSE);
 	if ((mov->compress_flags & GF_ISOM_COMP_FORCE_ALL) || (comp_size + COMP_BOX_COST_BYTES < box_size)) {
 		if (bs) {
 			if (use_cmov) {
@@ -569,7 +569,7 @@ u64 GetMoovAndMetaSize(GF_ISOFile *movie, GF_List *writers)
 		}
 
 		gf_bs_get_content(bs, &box_data, &box_size);
-		gf_gz_compress_payload_ex(&box_data, box_size, &osize, 0, GF_FALSE, NULL, GF_TRUE);
+		gf_gz_compress_payload_ex(&box_data, box_size, &osize, 0, GF_FALSE, NULL, GF_FALSE);
 		gf_free(box_data);
 		gf_bs_del(bs);
 		return osize + 8 + 8 + 12 + 12;

src/isomedia/track.c

@@ -211,7 +211,7 @@ GF_Err GetESD(GF_MovieBox *moov, GF_ISOTrackID trackID, u32 StreamDescIndex, GF_
 				esd->decoderConfig->rvc_config = (GF_DefaultDescriptor *) gf_odf_desc_new(GF_ODF_DSI_TAG);
 				if (mime_type && !strcmp(mime_type, "application/rvc-config+xml+gz") ) {
 #if !defined(GPAC_DISABLE_ZLIB)
-					gf_gz_decompress_payload(rvc_cfg_data, rvc_cfg_size, &esd->decoderConfig->rvc_config->data, &esd->decoderConfig->rvc_config->dataLength);
+					gf_gz_decompress_payload_ex(rvc_cfg_data, rvc_cfg_size, &esd->decoderConfig->rvc_config->data, &esd->decoderConfig->rvc_config->dataLength, GF_TRUE);
 					gf_free(rvc_cfg_data);
 #endif
 				} else {

src/media_tools/dsmcc.c

@@ -610,7 +610,7 @@ static GF_Err dsmcc_module_complete(GF_M2TS_DSMCC_OVERLORD* dsmcc_overlord,GF_M2
 				u32 uncomp_size;
 				u8* uncompressed_data;
 
-				gf_gz_decompress_payload(dsmcc_module->buffer,dsmcc_module->byte_sift, &uncompressed_data, &uncomp_size);
+				gf_gz_decompress_payload_ex(dsmcc_module->buffer,dsmcc_module->byte_sift, &uncompressed_data, &uncomp_size, GF_TRUE);
 				//dsmcc_process_biop_data(dsmcc_overlord,dsmcc_module,uncompressed_data,dsmcc_module->original_size);
 
 				if(dsmcc_module->original_size == uncomp_size) {

src/media_tools/route_dmx.c

@@ -1362,7 +1362,7 @@ static GF_Err gf_route_dmx_process_service_signaling(GF_ROUTEDmx *routedmx, GF_R
 		}
 		memcpy(routedmx->buffer, object->payload, object->total_length);
 		raw_size = routedmx->unz_buffer_size;
-		e = gf_gz_decompress_payload(routedmx->buffer, object->total_length, &routedmx->unz_buffer, &raw_size);
+		e = gf_gz_decompress_payload_ex(routedmx->buffer, object->total_length, &routedmx->unz_buffer, &raw_size, GF_TRUE);
 		if (e) {
 			GF_LOG(GF_LOG_ERROR, GF_LOG_ROUTE, ("[ROUTE] Service %d failed to decompress signaling bundle: %s\n", s->service_id, gf_error_to_string(e) ));
 			return e;
@@ -1747,7 +1747,7 @@ static GF_Err gf_route_dmx_process_lls(GF_ROUTEDmx *routedmx)
 		return GF_OK;
 	}
 
-	e = gf_gz_decompress_payload(&routedmx->buffer[4], read-4, &routedmx->unz_buffer, &raw_size);
+	e = gf_gz_decompress_payload_ex(&routedmx->buffer[4], read-4, &routedmx->unz_buffer, &raw_size, GF_TRUE);
 	if (e) {
 		GF_LOG(GF_LOG_ERROR, GF_LOG_ROUTE, ("[ROUTE] Failed to decompress %s table: %s\n", name, gf_error_to_string(e) ));
 		return e;

src/scene_manager/scene_engine.c

@@ -400,7 +400,7 @@ static GF_Err gf_seng_encode_dims_au(GF_SceneEngine *seng, u16 ESID, GF_List *co
 	if (compress_dims) {
 #ifndef GPAC_DISABLE_ZLIB
 		dims_header |= GF_DIMS_UNIT_C;
-		e = gf_gz_compress_payload(&buffer, buffer_len, &buffer_len);
+		e = gf_gz_compress_payload_ex(&buffer, buffer_len, &buffer_len, 0, GF_FALSE, NULL, GF_TRUE);
 		GF_LOG(GF_LOG_DEBUG, GF_LOG_SCENE, ("/ compressed (%d)", buffer_len));
 		if (e) goto exit;
 #else

src/utils/base_encoding.c

@@ -199,9 +199,9 @@ GF_Err gf_gz_compress_payload_ex(u8 **data, u32 data_len, u32 *max_size, u8 data
 	stream.opaque = (voidpf)NULL;
 
 	if (use_gz) {
-		err = deflateInit(&stream, 9);
-	} else {
 		err = deflateInit2(&stream, 9, Z_DEFLATED, 16+MAX_WBITS, 8, Z_DEFAULT_STRATEGY);
+	} else {
+		err = deflateInit(&stream, 9);
 	}
 
 	if (err != Z_OK) {
@@ -274,9 +274,9 @@ GF_Err gf_gz_decompress_payload_ex(u8 *data, u32 data_len, u8 **uncompressed_dat
 	d_stream.avail_out = size;
 
 	if (use_gz) {
-		err = inflateInit(&d_stream);
-	} else {
 		err = inflateInit2(&d_stream, 16+MAX_WBITS);
+	} else {
+		err = inflateInit(&d_stream);
 	}
 
 	if (err == Z_OK) {