Issue 5146 - Implement DownloadableSubscription parsing in C++

compiled/subscription/DownloadableSubscription.cpp

Index: compiled/subscription/DownloadableSubscription.cpp
===================================================================
--- a/compiled/subscription/DownloadableSubscription.cpp
+++ b/compiled/subscription/DownloadableSubscription.cpp
@@ -10,26 +10,284 @@
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <cwctype>
+#include <limits>
+
 #include "DownloadableSubscription.h"
+#include "../Base64.h"
+#include "../FilterNotifier.h"
+#include "../StringScanner.h"
+#include "../filter/CommentFilter.h"
+
+namespace {
+  constexpr int MILLIS_IN_HOUR = 60 * 60 * 1000;
+  constexpr int MILLIS_IN_DAY = 24 * MILLIS_IN_HOUR;
+  // limits
+  constexpr int64_t MAX_HOUR = std::numeric_limits<int64_t>::max() / MILLIS_IN_HOUR;
+  constexpr int64_t MAX_DAY = std::numeric_limits<int64_t>::max() / MILLIS_IN_DAY;
+
+  typedef std::pair<DependentString, DependentString> Param;
+
+  Param ParseParam(const String& text)
+  {
+    Param param;
+
+    if (text[0] == u'!')
+    {
+      bool foundColon = false;
+      String::size_type beginParam = 0;
+      String::size_type endParam = 0;
+      String::size_type beginValue = 0;
+      for (String::size_type i = 1; i < text.length(); i++)
+      {
+        switch (text[i])
+        {
+        case ' ':
+        case '\t':
+          if (beginParam > 0 && !foundColon)
+          {
+            endParam = i;
+          }
+          break;
+        case ':':
+          foundColon = true;
+          endParam = i;
+          break;
+        default:
+          if (foundColon)
+          {
+            beginValue = i;
+          }
+          else
+          {
+            if (beginParam == 0)
+              beginParam = i;
+          }
+          break;
+        }
+        if (beginValue > 0)
+          break;
+      }
+      if (beginValue > 0)
+      {
+        param.first = DependentString(text, beginParam, endParam - beginParam);
+        param.first.toLower();
+        param.second = DependentString(
+          text, beginValue, text.length() - beginValue);
+      }
+    }
+    return param;
+  }
+}
+
+DownloadableSubscription_Parser::DownloadableSubscription_Parser()
+  : mFirstLine(true)
+{
+  annotate_address(this, "DownloadableSubscription_Parser");
+}
+
+namespace {
+  const DependentString ADBLOCK_HEADER(u"[Adblock"_str);
+
+  // Only check for trailing base64 padding. There should be at most 2 '='.
+  // In that case return a truncated string.
+  DependentString CleanUpChecksum(const String& checksum)
+  {
+    const auto len = checksum.length();
+    if ((len > 22 && len <= 24) &&
+        (checksum[22] == u'=' && (len == 23 || checksum[23] == u'=')))
+      return DependentString(checksum, 0, 22);
+    return DependentString(checksum);
+  }
+}
+
+void DownloadableSubscription_Parser::Process(const String& line)
+{
+  bool isHeader = false;
+  bool doChecksum = true;
+  isHeader = line.find(ADBLOCK_HEADER) != String::npos;
+  if (!isHeader)
+  {
+    auto param = ParseParam(line);
+    if (param.first.is_invalid())
+      mFiltersText.emplace_back(line);
+    else
+    {
+      if (param.first == u"checksum"_str)
+      {
+        mParams[param.first] = CleanUpChecksum(param.second);
+        doChecksum = false;
+      }
+      else
+        mParams[param.first] = param.second;
+    }
+  }
+  // Checksum is an MD5 checksum (base64-encoded without the trailing "=") of
+  // all lines in UTF-8 without the checksum line, joined with "\n".
+  if (doChecksum)
+  {
+    if (!mFirstLine)
+      mChecksum.Update((const uint8_t*)"\n", 1);
+    else
+      mFirstLine = false;
+    mChecksum.Update(line);
+  }
+}
+
+int64_t DownloadableSubscription_Parser::ParseExpires(const String& expires)
+{
+  bool isHour = false;
+  StringScanner scanner(expires);
+  String::size_type numStart = 0;
+  String::size_type numLen = 0;
+  while(!scanner.done())
+  {
+    auto ch = scanner.next();
+    if (std::iswdigit(ch))
+    {
+      if (numLen == 0)
+        numStart = scanner.position();
+      numLen++;
+    }
+    else if (std::iswspace(ch))
+    {
+      if (numLen)
+        break;
+    }
+    else
+    {
+      if (numLen)
+        scanner.back();
+      break;
+    }
+  }
+
+  DependentString numStr(expires, numStart, numLen);
+  int64_t num = numStr.toInt<int64_t>();
+  if (num == 0)
+    return 0;
+
+  while (!scanner.done())
+  {
+    auto ch = scanner.next();
+    if (std::iswspace(ch))
+      continue;
+
+    if (ch == u'h')
+      isHour = true;
+
+    // assume we are done here. The rest is ignored.
+    break;
+  }
+  // check for overflow.
+  if ((isHour && (num > MAX_HOUR)) || (num > MAX_DAY))
+    return 0;
+
+  num *= isHour ? MILLIS_IN_HOUR : MILLIS_IN_DAY;
+  return num;
+}
+
+bool DownloadableSubscription_Parser::VerifyChecksum()
+{
+  if (!mParams.find(u"checksum"_str))
+    return true;
+
+  if (mB64Checksum.is_invalid())
+  {
+    uint8_t checksum[MD5::CHECKSUM_LENGTH];
+    mChecksum.Final(checksum);
+    mB64Checksum = ToBase64(checksum, MD5::CHECKSUM_LENGTH);
+  }
+  return (mParams[u"checksum"_str] == mB64Checksum);
+}
+
+int64_t DownloadableSubscription_Parser::Finalize(DownloadableSubscription& subscription)
+{
+  if (mB64Checksum.is_invalid())
+    VerifyChecksum(); // here we ignore the checksum, but we calculate it.
+
+  auto entry = mParams.find(u"title"_str);
+  if (entry)
+  {
+    subscription.SetTitle(entry->second);
+    subscription.SetFixedTitle(true);
+  }
+  else
+    subscription.SetFixedTitle(false);
+
+  int32_t version = 0;
+  entry = mParams.find(u"version"_str);
+  if (entry)
+    version = entry->second.toInt<int32_t>();
+  subscription.SetDataRevision(version);
+
+  int64_t expires = 0;
+  entry = mParams.find(u"expires"_str);
+  if (entry)
+    expires = ParseExpires(entry->second);
+
+  FilterNotifier::SubscriptionChange(
+    FilterNotifier::Topic::SUBSCRIPTION_BEFORE_FILTERS_REPLACED,
+    subscription);
+
+  Subscription::Filters filters;
+  filters.reserve(mFiltersText.size());
+  for (auto text : mFiltersText)
+  {
+    DependentString dependent(text);
+    filters.emplace_back(Filter::FromText(dependent), false);
+  }
+
+  subscription.SetFilters(std::move(filters));
+  FilterNotifier::SubscriptionChange(
+    FilterNotifier::Topic::SUBSCRIPTION_FILTERS_REPLACED, subscription);
+
+  return expires;
+}
+
+namespace {
+  DependentString emptyString = u""_str;
+}
+
+const String& DownloadableSubscription_Parser::GetRedirect() const
+{
+  auto entry = mParams.find(u"redirect"_str);
+  if (entry)
+    return entry->second;
+  return emptyString;
+}
+
+const String& DownloadableSubscription_Parser::GetHomepage() const
+{
+  auto entry = mParams.find(u"homepage"_str);
+  if (entry)
+    return entry->second;
+  return emptyString;
+}
 
 DownloadableSubscription::DownloadableSubscription(const String& id)
     : Subscription(classType, id), mFixedTitle(false), mLastCheck(0),
       mHardExpiration(0), mSoftExpiration(0), mLastDownload(0), mLastSuccess(0),
       mErrorCount(0), mDataRevision(0), mDownloadCount(0)
 {
   SetTitle(id);
 }
 
+DownloadableSubscription_Parser* DownloadableSubscription::ParseDownload()
+{
+  return new DownloadableSubscription_Parser();
+}
+
 OwnedString DownloadableSubscription::Serialize() const
 {
   OwnedString result(Subscription::Serialize());
   if (mFixedTitle)
     result.append(u"fixedTitle=true\n"_str);
   if (!mHomepage.empty())
   {
     result.append(u"homepage="_str);