Add EMM filtering support.

ChangeLog

@@ -13,6 +13,7 @@
  * Add support for multiple CAMD addresses returned when resolving CAMD
    server hostname. tsdecrypt would try each of the addresses and use
    the one that works.
+ * Add --emm-filter (-a) option. This option implements EMM filtering.
 
 2012-04-19 : Version 8.1
  * Add support for Bulcrypt CAS.

FILTERING

@@ -0,0 +1,113 @@
+tsdecrypt EMM filtering support
+===============================
+tsdecrypt have no specific knowledge about EMMs of each of the
+supported CA systems. This allows tsdecrypt to work even when
+the CA system is unknown. It just sends the EMM/ECM streams to the
+CAMD server for processing and filtering. The lack of specific
+knowledge about each of the supported CA systems is a feature.
+
+However there are cases where limiting the number of EMMs that
+reach CAMD is a must have feature. Since there is no code in
+tsdecrypt to detect whether the EMM type is GLOBAL, SHARED,
+UNIQUE, there is no detection of provider IDs, channel IDs or
+card numbers a simpler approach to filtering was implemented.
+
+The basic idea was implemented by Julian Gardner in his emm-buffer-mask
+patches. These patches were never merged in tsdecrypt but they
+inspired the current filtering implementation.
+
+tsdecrypt's EMM filters describe whether EMM should be processed
+or not based on an offset and data bytes that are compared with the
+incoming EMM.
+
+The option responsible for defining EMM filters is --emm-filter (-a)
+followed by the filter definition. Up to 16 filters can be defined.
+
+Filter definitions
+==================
+Filter definition contain command and command settings. The command
+and the settings are separated by / (forward slash) symbol, the
+settings are separated by ^ symbol.
+
+  Command  Settings
+  -------  --------
+  command[/setting1=abc,setting2=xyz...]
+
+Since filter definitions are passed as command line parameters, you must
+ensure that they are enclosed in quotes if they contain spaces.
+
+Filter commands
+===============
+Currently defined commands are:
+
+   accept_all    - Set the default to allow all EMMs to reach CAMD except
+                   EMMs that match "reject" command.
+
+                   When no "accept_all" or "reject_all" commands were used it
+                   is assumed that "accept_all" was the first command.
+
+   reject_all    - Set the default to skip all EMMs except those that
+                   are accepted by "accept" command.
+
+   * Both "accept_all" and "reject_all" can be used without command settings.
+
+   accept        - This command instructs tsdecrypt to allow EMM that matches
+                   the definition to be processed.
+
+   reject        - This command instructs tsdecrypt to skip EMM that matches
+                   the definition to be processed.
+
+   * Both commands must have at least two settings (offset and data).
+
+Filter settings
+===============
+Currently defined command settings are:
+
+ * name=X        - Sets the filter name (used when displaying filters).
+                   The name can not be longer than 32 symbols.
+
+ * offset=X      - Sets the offset at which data bytes would be checked
+                   against EMM. The default offset is 0.
+
+ * data=XX YY ZZ - Data bytes are series of hex numbers separated by
+                   space " " or dot ".". When data bytes are processed 0x
+                   prefix is removed from them. The maximum data bytes is 16.
+
+Filter processing
+=================
+Filters are processed one by one until "accept" or "reject" filter matches.
+If no "accept" or "reject" filters match, then the default match determined
+by "accept_all" or "reject_all" is returned.
+
+Example filters
+===============
+
+  Bulcrypt white list example:
+    Accept all EMMs except those starting with 8a 70 b4, 8b 70 b4, 8c or 8d.
+
+       tsdecrypt ... \
+         --emm-filter accept_all \
+         --emm-filter "reject/name=Bulcrypt_unknown_EMMs,offset=0,data=8a 70 b4" \
+         --emm-filter "reject/name=Bulcrypt_unknown_EMMs,offset=0,data=8b 70 b4" \
+         --emm-filter "reject/name=Some EMM,data=0x8c" \
+         --emm-filter reject/data=0x8d \
+
+  Bulcrypt black list example:
+    Reject all EMMs that don't start with 82, 84 or 85.
+
+      tsdecrypt ... \
+        --emm-filter reject_all \
+        --emm-filter accept/name=Bulcrypt_EMMs,offset=0,data=82 \
+        --emm-filter accept/name=Bulcrypt_EMMs,data=84 \
+        --emm-filter accept/name=Bulcrypt_EMMs,data=85 \
+
+  Example combination that builds white/black list.
+    Accept EMMs that start with 8a and 8b also accept EMMs which have
+    70 b4 at offset 1, and reject everything else.
+
+      tsdecrypt ... \
+        --emm-filter accept_all \
+        --emm-filter reject/name=Bulcrypt_unknown_EMMs,data=8a \
+        --emm-filter reject/name=Bulcrypt_unknown_EMMs,data=8b \
+        --emm-filter reject_all \
+        --emm-filter "accept/name=Bulcrypt_EMMs_with_correct_size,offset=1,data=70 b4" \

Makefile

@@ -41,6 +41,7 @@ tsdecrypt_SRC = data.c \
  csa.c \
  udp.c \
  util.c \
+ filter.c \
  camd.c \
  camd-cs378x.c \
  camd-newcamd.c \

README

@@ -164,6 +164,9 @@ EMM options:
  -f --emm-report-time <sec> | Report each <sec> seconds how much EMMs have been
                             .   received/processed. Set <sec> to 0 to disable
                             .   the reports. Default: 60 sec
+ -a --emm-filter <filter>   | Add EMM filter defined by <filter>.
+                            . This option can be used multiple times (max:16).
+                            . See FILTERING file for more info.
 
 ECM options:
  -X --ecm-pid <pid>         | Force ECM pid. Default: none

TODO

@@ -1,3 +1,2 @@
 - Sort nanos of viaccess and cryptoworks emms.
 - Add ipv6 support (multicast send/recv).
-- Investigate EMM filters implemented in https://github.com/joolzg/tsdecrypt

data.h

@@ -185,6 +185,26 @@ struct packet_buf {
 	uint8_t		data[188];
 };
 
+#define MAX_FILTERS     16
+#define MAX_FILTER_NAME 32
+#define MAX_FILTER_LEN  16
+
+enum filter_action {
+	FILTER_NO_MATCH   = 0,
+	FILTER_ACCEPT_ALL = 1,
+	FILTER_REJECT_ALL = 2,
+	FILTER_ACCEPT     = 3,
+	FILTER_REJECT     = 4,
+};
+
+struct filter {
+	enum filter_action action;			// If set to 1, the filter is negative else the filter is positive
+	uint8_t		offset;					// Offset into EMM
+	uint8_t		data_len;				// Filter length
+	uint8_t		data[MAX_FILTER_LEN];	// Filter bytes
+	char		name[MAX_FILTER_NAME];	// Filter name (default: NO_NAME)
+};
+
 #define MAX_PIDS 8192
 
 struct ts {
@@ -294,6 +314,9 @@ struct ts {
 
 	unsigned int		input_buffer_time;
 	LIST				*input_buffer;
+
+	int					emm_filters_num;
+	struct filter		emm_filters[MAX_FILTERS];
 };
 
 void data_init(struct ts *ts);

filter.c

@@ -0,0 +1,142 @@
+/*
+ * Filtering functions
+ * Copyright (C) 2012 Unix Solutions Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License (COPYING file) for more details.
+ *
+ */
+#include <ctype.h>
+#include <string.h>
+
+#include "data.h"
+#include "filter.h"
+
+int filter_parse(char *filter_def, struct filter *filter) {
+	int i, j, k, ret = 0;
+	char *str1, *saveptr1;
+	char *f = strdup(filter_def);
+	memset(filter, 0, sizeof(struct filter));
+	snprintf(filter->name, sizeof(filter->name), "NONAME");
+	for (j = 1, str1 = f; ; j++, str1 = NULL) {
+		char *token = strtok_r(str1, "/", &saveptr1);
+		if (token == NULL)
+			break;
+		if (j == 1) { // First token, the command
+			if (strstr(token, "accept_all") == token || strstr(token, "acceptall") == token) {
+				filter->action = FILTER_ACCEPT_ALL;
+				ret = 1;
+				goto OUT; // Other tokens are not needed
+			} else if (strstr(token, "reject_all") == token || strstr(token, "rejectall") == token) {
+				filter->action = FILTER_REJECT_ALL;
+				ret = 1;
+				goto OUT; // Other tokens are not needed
+			} else if (strstr(token, "accept") == token) {
+				filter->action = FILTER_ACCEPT;
+				continue; // Continue looking for other tokes
+			} else if (strstr(token, "reject") == token) {
+				filter->action = FILTER_REJECT;
+				continue; // Continue looking for other tokes
+			} else {
+				fprintf(stderr, "ERROR: Unknown filter command: %s\n", token);
+				ret = 0;
+				goto OUT; // Other tokens are not needed
+			}
+		}
+		if (j == 2) { // Second token, the settings
+			char *str2, *saveptr2;
+			for (k = 1, str2 = token; ; k++, str2 = NULL) {
+				char *token2 = strtok_r(str2, ",", &saveptr2);
+				if (token2 == NULL)
+					break;
+				char *eq = strrchr(token2, '=');
+				if (eq) {
+					if (strstr(token2, "ofs") == token2 || strstr(token2, "offset") == token2) {
+						filter->offset = strtoul(eq + 1, NULL, 0);
+					} else if (strstr(token2, "name") == token2) {
+						snprintf(filter->name, sizeof(filter->name), "%s", eq + 1);
+					} else if (strstr(token2, "data") == token2) {
+						char *data = eq + 1;
+						int len = strlen(data), consumed = 0;
+						for (i = 0; i < len; i++) { // Parse data (01 02 03 04 ...)
+							char ch = toupper(data[i]);
+							// Skip 0x prefixes
+							if (i + 1 < len && ch == '0' && toupper(data[i + 1]) == 'X')
+								continue;
+							if (!isxdigit(ch))
+								continue;
+							ch -= ch > 64 ? 55 : 48; // hex2dec
+							if (consumed % 2 == 0) {
+								filter->data[filter->data_len  ] += ch << 4;
+							} else {
+								filter->data[filter->data_len++] += ch;
+								if (filter->data_len + 1 >= MAX_FILTER_LEN) {
+									fprintf(stderr, "WARN : Too much filter data (max %d bytes), ignoring last bytes: %s\n",
+										MAX_FILTER_LEN, data + i + 2);
+									break;
+								}
+							}
+							consumed++;
+						}
+						ret = filter->data_len;
+					} else {
+						fprintf(stderr, "WARN : Unknown filter setting: %s\n", token2);
+					}
+				}
+			}
+		}
+	}
+OUT:
+	FREE(f);
+	return ret;
+}
+
+void filter_dump(struct filter *filter, char *buffer, unsigned int buf_len) {
+	unsigned int pos = 0;
+	memset(buffer, 0, buf_len);
+	pos += snprintf(buffer + pos, buf_len - pos, "Action: %s",
+		filter->action == FILTER_ACCEPT_ALL ? "ACCEPT_ALL (default)" :
+		filter->action == FILTER_REJECT_ALL ? "REJECT_ALL (default)" :
+		filter->action == FILTER_ACCEPT     ? "ACCEPT" :
+		filter->action == FILTER_REJECT     ? "REJECT" : "???");
+	if (filter->action == FILTER_ACCEPT || filter->action == FILTER_REJECT)
+		pos += snprintf(buffer + pos, buf_len - pos, " Name: %-20s", filter->name);
+	if (filter->action == FILTER_ACCEPT || filter->action == FILTER_REJECT) {
+		char tmp[MAX_FILTER_LEN * 6];
+		ts_hex_dump_buf(tmp, sizeof(tmp), filter->data, filter->data_len, 0);
+		pos += snprintf(buffer + pos, buf_len - pos, " Offset: %2d Data: %s", filter->offset, tmp);
+	}
+}
+
+static enum filter_action filter_match(uint8_t *data, unsigned int data_len, struct filter *filter) {
+	if (filter->action == FILTER_ACCEPT_ALL || filter->action == FILTER_REJECT_ALL)
+		return filter->action;
+	if (filter->action == FILTER_ACCEPT || filter->action == FILTER_REJECT) {
+		if (filter->data_len + filter->offset > data_len)
+			return FILTER_NO_MATCH;
+		if (memcmp(data + filter->offset, filter->data, filter->data_len) == 0)
+			return filter->action;
+	}
+	return FILTER_NO_MATCH;
+}
+
+int filter_match_emm(struct ts *ts, uint8_t *data, unsigned int data_len) {
+	int i, ret = 1;
+	for (i = 0; i < ts->emm_filters_num; i++) {
+		enum filter_action result = filter_match(data, data_len, &ts->emm_filters[i]);
+		switch (result) {
+			case FILTER_NO_MATCH  : continue;
+			case FILTER_ACCEPT_ALL: ret = 1; continue;
+			case FILTER_ACCEPT    : ret = 1; break;
+			case FILTER_REJECT_ALL: ret = 0; continue;
+			case FILTER_REJECT    : ret = 0; break;
+		}
+	}
+	return ret;
+}

filter.h

@@ -0,0 +1,28 @@
+/*
+ * Filtering functions
+ * Copyright (C) 2012 Unix Solutions Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License (COPYING file) for more details.
+ *
+ */
+#ifndef FILTERS_H
+#define FILTERS_H
+
+#include <inttypes.h>
+
+#include "data.h"
+
+int filter_parse(char *filter_def, struct filter *filter);
+
+int filter_match_emm(struct ts *ts, uint8_t *data, unsigned int data_len);
+
+void filter_dump(struct filter *filter, char *buffer, unsigned int buf_len);
+
+#endif

tables.c

@@ -12,9 +12,12 @@
  * GNU General Public License (COPYING file) for more details.
  *
  */
+#include <string.h>
+
 #include "data.h"
 #include "tables.h"
 #include "camd.h"
+#include "filter.h"
 
 #include "libtsfuncs/tsfuncs.h"
 #include "libfuncs/libfuncs.h"
@@ -270,17 +273,28 @@ static void __process_emm(struct ts *ts, uint16_t pid, uint8_t *ts_packet) {
 
 	struct ts_header *th = &ts->emm->ts_header;
 	struct ts_section_header *sec = ts->emm->section_header;
+
+	int emm_ok = 1;
+	if (ts->emm_filters_num)
+		emm_ok = filter_match_emm(ts, sec->section_data, sec->section_data_len);
+
 	if (ts->debug_level >= 2) {
 		ts_hex_dump_buf(dump, dump_buf_sz, sec->section_data, min(dump_sz, sec->section_data_len), 0);
-		ts_LOGf("EMM | SID 0x%04x CAID: 0x%04x PID 0x%04x Table: 0x%02x Length: %4d Data: %s..\n",
+		ts_LOGf("EMM | SID 0x%04x CAID: 0x%04x PID 0x%04x Table: 0x%02x Length: %4d %s %s..\n",
 			ts->service_id,
 			ts->emm_caid,
 			th->pid,
 			sec->table_id,
 			sec->section_data_len,
+			emm_ok == 1 ? "Data:" : "SKIP:",
 			dump);
 	}
-	camd_process_packet(ts, camd_msg_alloc(EMM_MSG, ts->emm_caid, ts->service_id, sec->section_data, sec->section_data_len));
+
+	if (emm_ok)
+		camd_process_packet(ts, camd_msg_alloc(EMM_MSG, ts->emm_caid, ts->service_id, sec->section_data, sec->section_data_len));
+	else
+		ts->emm_skipped_count++;
+
 	ts_privsec_copy(ts->emm, ts->last_emm);
 	ts_privsec_clear(ts->emm);
 }

tsdecrypt.1

@@ -223,6 +223,17 @@ stream has very high rate and is interfering with ECM processing. Using
 Set interval for EMM reports. The default is \fB60\fR seconds. Set to \fB0\fR
 to disable EMM reports.
 .TP
+\fB\-a\fR, \fB\-\-emm\-filter\fR <filter_definition>
+Add EMM filter described by <filter_definition>. EMM filters are useful if
+you want to limit the number of EMMs that should reach your CAMD server.
+The basic \fB<filter_defintion>\fR is \fBCommand/Settings\fR where
+the commands are: \fBaccept_all\fR, \fBreject_all\fR, \fBaccept\fR
+and \fBreject\fR.
+
+For more information about filtering and for example filters, please
+read \fBFILTERING\fR file that comes with tsdecrypt. This option can be
+used multiple times to define up to \fB16\fR different filters.
+.TP
 .SH ECM OPTIONS
 .PP
 .TP

tsdecrypt.c

@@ -16,6 +16,7 @@
 #include <unistd.h>
 #include <getopt.h>
 #include <string.h>
+#include <ctype.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <signal.h>
@@ -35,6 +36,7 @@
 #include "process.h"
 #include "udp.h"
 #include "notify.h"
+#include "filter.h"
 
 #define FIRST_REPORT_SEC 3
 
@@ -78,7 +80,7 @@ static void LOG_func(const char *msg) {
 		LOG(msg);
 }
 
-static const char short_options[] = "i:d:N:Sl:L:F:I:RzM:T:W:O:o:t:rk:g:upwxyc:C:Y:Q:A:s:U:P:B:46eZ:Ef:X:H:G:KJ:D:jbhVn:m:";
+static const char short_options[] = "i:d:N:Sl:L:F:I:RzM:T:W:O:o:t:rk:g:upwxyc:C:Y:Q:A:s:U:P:B:46eZ:Ef:a:X:H:G:KJ:D:jbhVn:m:";
 
 // Unused short options: aqv01235789
 static const struct option long_options[] = {
@@ -126,6 +128,7 @@ static const struct option long_options[] = {
 	{ "emm-pid",			required_argument, NULL, 'Z' },
 	{ "emm-only",			no_argument,       NULL, 'E' },
 	{ "emm-report-time",	required_argument, NULL, 'f' },
+	{ "emm-filter",			required_argument, NULL, 'a' },
 
 	{ "ecm-pid",			required_argument, NULL, 'X' },
 	{ "ecm-report-time",	required_argument, NULL, 'H' },
@@ -219,6 +222,9 @@ static void show_help(struct ts *ts) {
 	printf(" -f --emm-report-time <sec> | Report each <sec> seconds how much EMMs have been\n");
 	printf("                            .   received/processed. Set <sec> to 0 to disable\n");
 	printf("                            .   the reports. Default: %d sec\n", ts->emm_report_interval);
+	printf(" -a --emm-filter <filter>   | Add EMM filter defined by <filter>.\n");
+	printf("                            . This option can be used multiple times (max:%u).\n", MAX_FILTERS);
+	printf("                            . See FILTERING file for more info.\n");
 	printf("\n");
 	printf("ECM options:\n");
 	printf(" -X --ecm-pid <pid>         | Force ECM pid. Default: none\n");
@@ -513,6 +519,19 @@ static void parse_options(struct ts *ts, int argc, char **argv) {
 					ts->emm_report_interval = 86400;
 				break;
 
+			case 'a': // --emm-filter
+				if (ts->emm_filters_num + 1 > MAX_FILTERS) {
+					fprintf(stderr, "ERROR: Maximum allowed filters are %d.\n", MAX_FILTERS);
+					exit(EXIT_FAILURE);
+				}
+				if (filter_parse(optarg, &ts->emm_filters[ts->emm_filters_num])) {
+					ts->emm_filters_num++;
+				} else {
+					fprintf(stderr, "ERROR: Can't parse EMM filter: %s\n", optarg);
+					exit(EXIT_FAILURE);
+				}
+				break;
+
 			case 'X': // --ecm-pid
 				ts->forced_ecm_pid = strtoul(optarg, NULL, 0) & 0x1fff;
 				break;
@@ -598,6 +617,7 @@ static void parse_options(struct ts *ts, int argc, char **argv) {
 		ts->cw_warn_sec = 0;
 		ts->camd.no_reconnect = 1;
 		ts->camd.check_emm_errors = 1;
+		ts->emm_filters_num = 0;
 	}
 
 	// Constant codeword is special. Disable conflicting options
@@ -755,6 +775,11 @@ static void parse_options(struct ts *ts, int argc, char **argv) {
 		ts_LOGf("TS discont : %s\n", ts->ts_discont ? "report" : "ignore");
 		ts->threaded = !(ts->input.type == FILE_IO && ts->input.fd != 0);
 	}
+
+	if (!packet_from_file && !ts->emm_only) {
+		ts_LOGf("Decoding   : %s\n", ts->threaded ? "threaded" : "single thread");
+	}
+
 	if (!packet_from_file) {
 		if (ts->emm_send && ts->emm_report_interval)
 			ts_LOGf("EMM report : %d sec\n", ts->emm_report_interval);
@@ -773,6 +798,14 @@ static void parse_options(struct ts *ts, int argc, char **argv) {
 		}
 	}
 
+	if (ts->emm_send) {
+		for (i = 0; i < ts->emm_filters_num; i++) {
+			char tmp[512];
+			filter_dump(&ts->emm_filters[i], tmp, sizeof(tmp));
+			ts_LOGf("EMM filter : [%2d] %s\n", i + 1,  tmp);
+		}
+	}
+
 	if (!packet_from_file) {
 		if (!ts->emm_only && ts->ecm_report_interval)
 			ts_LOGf("ECM report : %d sec\n", ts->emm_report_interval);