Initial libadblockplus integration

Shared/PluginFilter.cpp

Index: Shared/PluginFilter.cpp
===================================================================
--- a/Shared/PluginFilter.cpp
+++ b/Shared/PluginFilter.cpp
@@ -80,14 +80,13 @@
 // CFilterElementHide
 // ============================================================================
 
-CFilterElementHide::CFilterElementHide(const CString& filterText, const CString& filterFile) : m_filterText(filterText), m_filterFile(filterFile)
+CFilterElementHide::CFilterElementHide(const CString& filterText) : m_filterText(filterText)
 {
 }
 
 CFilterElementHide::CFilterElementHide(const CFilterElementHide& filter)
 {
   m_filterText = filter.m_filterText;
-  m_filterFile = filter.m_filterFile;
 
   m_tagId = filter.m_tagId;
   m_tagClassName = filter.m_tagClassName;
@@ -117,7 +116,6 @@
   m_stringElements = filter.m_stringElements;
 
   m_filterText = filter.m_filterText;
-  m_filterFile = filter.m_filterFile;
 
   m_domains = filter.m_domains;
   m_domainsNot = filter.m_domainsNot;
@@ -136,7 +134,7 @@
 // CPluginFilter
 // ============================================================================
 
-CPluginFilter::CPluginFilter(const TFilterFileList& list, const CString& dataPath) : m_dataPath(dataPath)
+CPluginFilter::CPluginFilter(const CString& dataPath) : m_dataPath(dataPath)
 {
   m_contentMap["document"] = CFilter::contentTypeDocument;
   m_contentMap["subdocument"] = CFilter::contentTypeSubdocument;
@@ -163,16 +161,11 @@
   m_contentMapText[CFilter::contentTypeSubdocument] = "iframe";
   m_contentMapText[CFilter::contentTypeStyleSheet] = "css";
 
-  ParseFilters(list); 
+  ClearFilters(); 
 }
 
 
-CPluginFilter::CPluginFilter(const CString& dataPath) : m_dataPath(dataPath)
-{
-}
-
-
-bool CPluginFilter::AddFilterElementHide(CString filterText, CString filterFile)
+bool CPluginFilter::AddFilterElementHide(CString filterText)
 {
   int delimiterPos = filterText.Find(L"#");
   if (delimiterPos < 0 || filterText.GetLength() <= delimiterPos + 1)
@@ -190,7 +183,7 @@
   s_criticalSectionFilterMap.Lock();    
   {
     // Create filter descriptor
-    CFilterElementHide filter(filterText, filterFile);
+    CFilterElementHide filter(filterText);
 
     CString filterDomains = filterText.Left(delimiterPos).MakeLower();
     CString filterString  = filterText.Right(filterText.GetLength() - delimiterPos - (isOldFormat ? 1 : 2));
@@ -644,22 +637,22 @@
                 if (!id.IsEmpty() && !classNames.IsEmpty())
                 {
                   DEBUG_HIDE_EL(indent + "HideEl::Found (domain) filter:" + filterIt->second.m_filterText)
-                    CPluginDebug::DebugResultHiding(tag, "id:" + id + " class:" + classNames, filterIt->second.m_filterText, filterIt->second.m_filterFile);
+                    CPluginDebug::DebugResultHiding(tag, "id:" + id + " class:" + classNames, filterIt->second.m_filterText);
                 }
                 else if (!id.IsEmpty())
                 {
                   DEBUG_HIDE_EL(indent + "HideEl::Found (domain) filter:" + filterIt->second.m_filterText)
-                    CPluginDebug::DebugResultHiding(tag, "id:" + id, filterIt->second.m_filterText, filterIt->second.m_filterFile);
+                    CPluginDebug::DebugResultHiding(tag, "id:" + id, filterIt->second.m_filterText);
                 }
                 else if (!classNames.IsEmpty())
                 {
                   DEBUG_HIDE_EL(indent + "HideEl::Found (domain) filter:" + filterIt->second.m_filterText)
-                    CPluginDebug::DebugResultHiding(tag, "class:" + classNames, filterIt->second.m_filterText, filterIt->second.m_filterFile);
+                    CPluginDebug::DebugResultHiding(tag, "class:" + classNames, filterIt->second.m_filterText);
                 }
                 else
                 {
                   DEBUG_HIDE_EL(indent + "HideEl::Found (domain) filter:" + filterIt->second.m_filterText)
-                    CPluginDebug::DebugResultHiding(tag, "-" + classNames, filterIt->second.m_filterText, filterIt->second.m_filterFile);
+                    CPluginDebug::DebugResultHiding(tag, "-" + classNames, filterIt->second.m_filterText);
                 }
               }
 #endif
@@ -682,12 +675,12 @@
                     if (!id.IsEmpty())
                     {
                       DEBUG_HIDE_EL(indent + "HideEl::Found (domain) filter:" + filterIt->second.m_filterText)
-                        CPluginDebug::DebugResultHiding(tag, "id:" + id + " class:" + classNames, filterIt->second.m_filterText, filterIt->second.m_filterFile);
+                        CPluginDebug::DebugResultHiding(tag, "id:" + id + " class:" + classNames, filterIt->second.m_filterText);
                     }
                     else
                     {
                       DEBUG_HIDE_EL(indent + "HideEl::Found (domain) filter:" + filterIt->second.m_filterText)
-                        CPluginDebug::DebugResultHiding(tag, "-" + classNames, filterIt->second.m_filterText, filterIt->second.m_filterFile);
+                        CPluginDebug::DebugResultHiding(tag, "-" + classNames, filterIt->second.m_filterText);
                     }
                   }
 #endif
@@ -722,7 +715,7 @@
         if (isHidden)
         {
           DEBUG_HIDE_EL(indent + "HideEl::Found (tag/id) filter:" + idIt->second.m_filterText)
-            CPluginDebug::DebugResultHiding(tag, "id:" + id, idIt->second.m_filterText, idIt->second.m_filterFile);
+            CPluginDebug::DebugResultHiding(tag, "id:" + id, idIt->second.m_filterText);
         }
 #endif
       }
@@ -738,7 +731,7 @@
           if (isHidden)
           {
             DEBUG_HIDE_EL(indent + "HideEl::Found (?/id) filter:" + idIt->second.m_filterText)
-              CPluginDebug::DebugResultHiding(tag, "id:" + id, idIt->second.m_filterText, idIt->second.m_filterFile);
+              CPluginDebug::DebugResultHiding(tag, "id:" + id, idIt->second.m_filterText);
           }
 #endif
         }
@@ -761,7 +754,7 @@
           if (isHidden)
           {
             DEBUG_HIDE_EL(indent + "HideEl::Found (tag/class) filter:" + classIt->second.m_filterText)
-              CPluginDebug::DebugResultHiding(tag, "class:" + className, classIt->second.m_filterText, classIt->second.m_filterFile);
+              CPluginDebug::DebugResultHiding(tag, "class:" + className, classIt->second.m_filterText);
           }
 #endif
         }
@@ -777,7 +770,7 @@
             if (isHidden)
             {
               DEBUG_HIDE_EL(indent + "HideEl::Found (?/class) filter:" + classIt->second.m_filterText)
-                CPluginDebug::DebugResultHiding(tag, "class:" + className, classIt->second.m_filterText, classIt->second.m_filterFile);
+                CPluginDebug::DebugResultHiding(tag, "class:" + className, classIt->second.m_filterText);
             }
 #endif
           }
@@ -799,7 +792,7 @@
         if (isHidden)
         {
           DEBUG_HIDE_EL(indent + "HideEl::Found (tag) filter:" + tagIt->second.m_filterText)
-            CPluginDebug::DebugResultHiding(tag, "-", tagIt->second.m_filterText, tagIt->second.m_filterFile);
+            CPluginDebug::DebugResultHiding(tag, "-", tagIt->second.m_filterText);
         }
 #endif
       }
@@ -810,646 +803,74 @@
   return isHidden;
 }
 
-
-void CPluginFilter::AddFilter(CString filterString, CString filterFile, int filterType)
-{
-  //We don't accept too short filters. Those are suspicious.
-  if (filterString.GetLength() < 5)
-    return;
-
-  CString raw = filterString;
-
-  // Here we should find a key for the filter
-  // We find a string of max 8 characters that does not contain any wildcards and which are unique for the filter
-
-  // Find settings part, identified by $
-  CString filterSettings;
-
-  int pos = filterString.Find('$');
-  if (pos > 0)
-  {
-    filterSettings = filterString.Right(filterString.GetLength() - pos - 1);
-    filterString = filterString.Left(pos);
-  }
-
-  // Split filterString to parts
-
-  bool bCheckFromStartDomain = false;
-  if (filterString.Find(L"||") == 0)
-  {
-    bCheckFromStartDomain = true;
-    filterString = filterString.Right(filterString.GetLength() - 2);
-  }
-
-  bool bCheckFromStart = false;
-  if (filterString.GetAt(0) == '|')
-  {
-    bCheckFromStart = true;
-    filterString = filterString.Right(filterString.GetLength() - 1);
-  }
-
-  bool bCheckFromEnd = false;
-  if (filterString.Right(1) == "|")
-  {
-    bCheckFromEnd = true;
-    filterString = filterString.Left(filterString.GetLength() - 1);
-  }
-
-  std::vector<CString> filterParts;
-  pos = 0;
-
-  while ((pos = filterString.Find('*')) >= 0)
-  {
-    if (pos > 0)
-    {
-      filterParts.push_back(filterString.Left(pos));
-    }
-    filterString = filterString.Right(filterString.GetLength() - pos - 1);
-  }
-
-  if (!filterString.IsEmpty())
-  {
-    filterParts.push_back(filterString);
-  }
-
-  TFilterMap* filterMap = m_filterMap[filterType];
-
-  // Define hash key
-  DWORD dwKey = 0;
-  DWORD dwKeyMap = 0;
-
-  int keyLength = 4;
-  int startCharacter = 0;
-  if (filterParts.size() < 1)
-  {
-    return;
-  }
-  CString filterPart = filterParts[0];
-  filterPart.MakeLower();
-
-  int nFilterParts = filterParts.size();
-  int nFilterPart = 0;
-
-  int filterPartLength = filterPart.GetLength();
-
-  if (filterPartLength >= 7)
-  {
-    if (filterPart.Find(L"http://") == 0)
-    {
-      startCharacter = 7;
-    }
-    else if (filterPart.Find(L"https://") == 0)
-    {
-      startCharacter = 8;
-    }
-  }
-
-  while (true)
-  {
-    // Part is too short as unique key? - try next filter part
-    while (filterPartLength < startCharacter + keyLength)
-    {
-      nFilterPart++;
-      if (nFilterPart >= nFilterParts)
-      {
-        break;
-      }
-
-      filterPart = filterParts[nFilterPart];
-      filterPart.MakeLower();
-      filterPartLength = filterPart.GetLength();
-
-      startCharacter = 0;
-    }
-
-    if (nFilterPart >= nFilterParts)
-    {
-      break;
-    }
-
-    int posSpecial = filterPart.Find('^', startCharacter);
-    if (posSpecial >= 0 && posSpecial < startCharacter + keyLength)
-    {
-      startCharacter = posSpecial + 1;
-    }
-    else
-    {
-      // Try key
-      DWORD dwTestKey = (filterPart.GetAt(startCharacter) << 24) | (filterPart.GetAt(startCharacter+1) << 16) | (filterPart.GetAt(startCharacter+2) << 8) | filterPart.GetAt(startCharacter+3);
-
-      // Now we have a substring which we can check
-      if (filterMap[0].find(dwTestKey) == filterMap[0].end()) 
-      {
-        dwKey = dwTestKey;
-        dwKeyMap = 0;
-        break;
-      }
-
-      // We already had a match - increment the start character
-      startCharacter++;
-    }
-  }
-
-  // Try second list
-  if (dwKey == 0)
-  {
-    dwKeyMap = 1;
-
-    startCharacter = 0;
-
-    filterPart = filterParts[0];
-    filterPart.MakeLower();
-    filterPartLength = filterPart.GetLength();
-
-    nFilterPart = 0;
-
-    if (filterPartLength >= 7)
-    {
-      if (filterPart.Find(L"http://") == 0)
-      {
-        startCharacter = 7;
-      }
-      else if (filterPart.Find(L"https://") == 0)
-      {
-        startCharacter = 8;
-      }
-    }
-
-    while (true)
-    {
-      // Part is too short as unique key? - try next filter part
-      while (filterPartLength < startCharacter + keyLength)
-      {
-        nFilterPart++;
-        if (nFilterPart >= nFilterParts)
-        {
-          break;
-        }
-
-        filterPart = filterParts[nFilterPart];
-        filterPart.MakeLower();
-        filterPartLength = filterPart.GetLength();
-
-        startCharacter = 0;
-      }
-
-      if (nFilterPart >= nFilterParts)
-      {
-        break;
-      }
-
-      int posSpecial = filterPart.Find('^', startCharacter);
-      if (posSpecial >= 0 && posSpecial < startCharacter + keyLength)
-      {
-        startCharacter = posSpecial + 1;
-      }
-      else
-      {
-        // Try key
-        DWORD dwTestKey = (filterPart.GetAt(startCharacter) << 24) | (filterPart.GetAt(startCharacter+1) << 16) | (filterPart.GetAt(startCharacter+2) << 8) | filterPart.GetAt(startCharacter+3);
-
-        // Now we have a substring which we can check
-        if (filterMap[1].find(dwTestKey) == filterMap[1].end()) 
-        {
-          dwKey = dwTestKey;
-          break;
-        }
-
-        // We already had a match - increment the start character
-        startCharacter++;
-      }
-    }
-  }
-
-  // Create the filter
-  CFilter filter;
-  filter.m_filterType = CFilter::filterTypeBlocking;
-  filter.m_stringElements = filterParts;
-  filter.m_isFromStart = bCheckFromStart;
-  filter.m_isFromStartDomain = bCheckFromStartDomain;
-  filter.m_isFromEnd = bCheckFromEnd;
-  filter.m_filterText = raw;
-  filter.m_filterFile = filterFile;
-
-  // Set content type of filter and other settings
-  if (!filterSettings.IsEmpty())
-  {
-    // Split filterSettings to parts
-    pos = 0;
-    bool hasContent = false;
-
-    while ((pos = filterSettings.Find(',')) >= 0 || !filterSettings.IsEmpty())
-    {
-      CString setting = pos >= 0 ? filterSettings.Left(pos) : filterSettings;
-      filterSettings = pos >= 0 ? filterSettings.Right(filterSettings.GetLength() - pos - 1) : "";
-
-      // Is content type negated
-      bool bNegate = false;
-      if (setting.GetAt(0) == '~')
-      {
-        bNegate = true;
-        setting = setting.Right(setting.GetLength() - 1);
-      }
-
-      // Apply content type
-      std::map<CString, int>::iterator it = m_contentMap.find(setting);
-      if (it != m_contentMap.end())
-      {
-        if (!hasContent)
-        {
-          if (bNegate)
-          {
-            filter.m_contentType = ~it->second;
-          }
-          else
-          {
-            filter.m_contentType = it->second;
-          }
-          hasContent = true;
-        }
-        else if (bNegate)
-        {
-          filter.m_contentType &= ~it->second;
-        }
-        else
-        {
-          filter.m_contentType |= it->second;
-        }
-      }
-      else if (setting == "match-case")
-      {
-        filter.m_isMatchCase = true;
-      }
-      else if (setting == "third-party")
-      {
-        if (bNegate)
-        {
-          filter.m_isFirstParty = true;
-        }
-        else
-        {
-          filter.m_isThirdParty = true;
-        }
-      }
-      else if (setting.Left(7) == "domain=")
-      {
-        int posDomain = 0;
-        setting = setting.Right(setting.GetLength() - 7);
-
-        while ((posDomain = setting.Find('|')) >= 0 || !setting.IsEmpty())
-        {
-          CString domain = posDomain >= 0 ? setting.Left(posDomain) : setting;
-          setting = posDomain >= 0 ? setting.Right(setting.GetLength() - posDomain - 1) : "";
-
-          if (domain.GetAt(0) == '~')
-          {
-            domain = domain.Right(domain.GetLength() - 1);
-
-            filter.m_domainsNot.insert(domain);
-          }
-          else
-          {
-            filter.m_domains.insert(domain);
-          }
-        }
-      }
-      else
-      {
-        DEBUG_FILTER("Filter::Error parsing filter:" + filterFile + "/" + raw + " (unhandled tag: " + setting + ")")
-          return;
-      }
-    }
-  }
-
-  // Add the filter
-  if (dwKey != 0)
-  {
-    filterMap[dwKeyMap][dwKey] = filter;
-  }
-  else
-  {
-    m_filterMapDefault[filterType].push_back(filter);
-  }
-}
-
-#ifdef PRODUCT_ADBLOCKPLUS
-
-bool CPluginFilter::DownloadFilterFile(const CString& url, const CString& filename)
-{
-  CString tempFile = CPluginSettings::GetTempFile(TEMP_FILE_PREFIX);
-
-  DEBUG_GENERAL("*** Downloading filter file:" + filename + " (to " + tempFile + ")");
-
-  bool bResult = !tempFile.IsEmpty();
-  if (bResult)
-  {
-    // if new filter urls are found download them and update the persistent data
-    HRESULT hr = ::URLDownloadToFile(NULL, url, tempFile, 0, NULL);
-    if (SUCCEEDED(hr))
-    {
-      CPluginFilterLock lock(filename);
-      if (lock.IsLocked())
-      {
-        // Move the temporary file to the new text file.
-        if (!::MoveFileEx(tempFile, CPluginSettings::GetDataPath(filename), MOVEFILE_REPLACE_EXISTING))
-        {
-          DWORD dwError = ::GetLastError();
-
-          // Not same device? copy/delete instead
-          if (dwError == ERROR_NOT_SAME_DEVICE)
-          {
-            if (!::CopyFile(tempFile, CPluginSettings::GetDataPath(filename), FALSE))
-            {
-              DEBUG_ERROR_LOG(::GetLastError(), PLUGIN_ERROR_FILTER, PLUGIN_ERROR_FILTER_COPY_FILE, "Filter::Unable to copy file:" + filename)
-
-                bResult = false;
-            }
-
-            ::DeleteFile(tempFile);
-          }
-          else
-          {
-            DEBUG_ERROR_LOG(dwError, PLUGIN_ERROR_FILTER, PLUGIN_ERROR_FILTER_MOVE_FILE, "Filter::Unable to replace file:" + filename)
-
-              bResult = false;
-          }
-        }
-      }
-      else
-      {
-        bResult = false;
-      }
-    }
-    else
-    {
-      DEBUG_ERROR_LOG(hr, PLUGIN_ERROR_FILTER, PLUGIN_ERROR_FILTER_DOWNLOAD_FILE, "Filter::Unable to download file: " + filename)
-
-        bResult = false;
-    }
-  }
-
-  return bResult;
-}
-
-#endif
-
-bool CPluginFilter::ReadFilter(const CString& filename, const CString& downloadPath)
+bool CPluginFilter::LoadHideFilters(std::vector<std::string> filters)
 {
   bool isRead = false;
 
 #ifdef PRODUCT_ADBLOCKPLUS
   CPluginClient* client = CPluginClient::GetInstance();
 #endif
-  CString fileContent;
 
-#ifdef PRODUCT_ADBLOCKPLUS
-  CPluginFilterLock lock(filename);
-  if (lock.IsLocked())
+  // Parse hide string
+  int pos = 0;
+
+  s_criticalSectionFilterMap.Lock();
   {
-#endif
-    DEBUG_GENERAL("*** Loading filter:" + m_dataPath + filename);
+    for (std::vector<std::string>::iterator it = filters.begin(); it < filters.end(); ++it)
+    {
+      CString filter((*it).c_str());
+      // If the line is not commented out
+      if (!filter.Trim().IsEmpty() && filter.GetAt(0) != '!' && filter.GetAt(0) != '[')

[Wladimir Palant, 2013/04/03 12:42:19]

I think you can assume that the filter engine will only give you actual hiding
filters, not comments or empty strings. There won't be any whitelists or
blocking rules either.

+      {
+        int filterType = 0;
 
-    //Rename old filter file if exists
-    if (filename == PERSONAL_FILTER_FILE)
-    {
-      HANDLE hFile = ::CreateFile(m_dataPath + PERSONAL_FILTER_FILE_OLD, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);  
-      if (hFile != INVALID_HANDLE_VALUE)
-      {
-        CloseHandle(hFile);
-        int res = _wrename(m_dataPath + PERSONAL_FILTER_FILE_OLD, m_dataPath + filename);
-        DWORD err = GetLastError();
-        err = 0;
-      }
-    }
+        // We need to categorize the filters
+        // We have three options, whitelist, block or element hiding
+        // See http://adblockplus.org/en/filters for further documentation
 
-    // Read file
-    HANDLE hFile = ::CreateFile(m_dataPath + filename, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);  
-    if (hFile == INVALID_HANDLE_VALUE)
-    {
-      DWORD dwError = ::GetLastError();
-#ifdef PRODUCT_ADBLOCKPLUS
-      // File not found - request another download!
-      if (dwError == ERROR_FILE_NOT_FOUND)
-      {
-        if (!downloadPath.IsEmpty())
+        // @@ indicates white listing rule
+        if (filter.Find(L"@@") == 0)
         {
-          client->RequestFilterDownload(filename, downloadPath);
+          filterType = CFilter::filterTypeWhiteList;
+
+          filter.Delete(0, 2);
         }
-        else if (filename == PERSONAL_FILTER_FILE)
+        // If a filter contains ## then it is a element hiding rule
+        else if (filter.Find(L"#") >= 0)
         {
-          // Open new file
-          HANDLE hPersonalFile = ::CreateFile(CPluginSettings::GetDataPath(PERSONAL_FILTER_FILE), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);  
-          if (hPersonalFile)
-          {
-            // Build filter string
-            CStringA line;
+          filterType = CFilter::filterTypeElementHide;
+        } 
+        //Anything we do not support here
+        else if (filter.Find(L"*") == 0)
+        {
+          filterType = CFilter::filterTypeUnknown;
+        }
+        // Else, it is a general rule
+        else
+        {
+          filterType = CFilter::filterTypeBlocking;
+        }
 
-            line += "[Adblock Plus - personal filters]\r\n";
-            line += "!\r\n";
-            line += "! In this file you can enter your own filters.\r\n";
-            line += "! Any updates to this file\r\n";
-            line += "! will take effect the next time a new tab or window is opened in\r\n";
-            line += "! Internet Explorer.\r\n";
-            line += "!\r\n";
-            line += "! To define filters you should use the adblockplus format, described on:\r\n";
-            line += "! http://adblockplus.org/en/filters\r\n";
-            line += "!\r\n";
-            line += "! File encoding: ANSI\r\n";
-            line += "!\r\n";
-            line += "!-------------------------Ad blocking rules--------------------------!\r\n";
-
-            // Write file
-            DWORD dwBytesWritten = 0;
-            ::WriteFile(hPersonalFile, line.GetBuffer(), line.GetLength(), &dwBytesWritten, NULL);
-
-            // Close file
-            ::CloseHandle(hPersonalFile);
+        try
+        {
+          if (filterType == CFilter::filterTypeElementHide)
+          { 
+            AddFilterElementHide(filter);
           }
         }
-      }
-      else
-#endif
-      {
-        DEBUG_ERROR_LOG(dwError, PLUGIN_ERROR_FILTER, PLUGIN_ERROR_FILTER_READ_FILE, "Filter::ParseFilters - Open file:" + filename)
-      }
-    }
-    else
-    {
-      // Read file
-      char buffer[8193];
-      LPVOID pBuffer = buffer;
-      LPBYTE pByteBuffer = (LPBYTE)pBuffer;
-      DWORD dwBytesRead = 0;
-      BOOL bRead = TRUE;
-
-      //Init MLang
-      CoInitialize(NULL);
-      HRESULT hr = S_OK;
-      CComPtr<IMultiLanguage2> pMultiLanguage;
-      hr = CoCreateInstance(CLSID_CMultiLanguage, NULL, CLSCTX_INPROC_SERVER, IID_IMultiLanguage2, (void**)&pMultiLanguage); 
-
-
-      bool codepageAvailable = false;
-      DetectEncodingInfo encodingInfos[10];
-      int scores = 10;
-      while ((bRead = ::ReadFile(hFile, pBuffer, 8192, &dwBytesRead, NULL)) == TRUE && dwBytesRead > 0)
-      {
-        //detect codepage based on first buffer
-        if (!codepageAvailable)
+        catch(...)
         {
-          unsigned int srcLength = fileContent.GetLength();
-          char* buf = (char*)fileContent.GetBufferSetLength(fileContent.GetLength());
-          hr = pMultiLanguage->DetectInputCodepage(0, 0, (char*)pBuffer, (int*)&dwBytesRead, encodingInfos, &scores);
-          codepageAvailable = true;
-        }
-
-
-
-        //Unicode
-        if ((encodingInfos[0].nCodePage == 1200) || (encodingInfos[0].nCodePage == 1201))
-        {
-          fileContent += CString((wchar_t*)buffer, dwBytesRead / 2);
-        }
-        else
-        {
-          pByteBuffer[dwBytesRead] = 0;
-          fileContent += buffer;
-        }
-      }
-
-      //Remove the BOM for UTF-8
-      if (((BYTE)fileContent.GetAt(0) == 0x3F) && ((BYTE)fileContent.GetAt(1) == 0xBB) && ((BYTE)fileContent.GetAt(2) == 0x57))
-      {
-        fileContent.Delete(0, 3);
-      }
-      // Read error        
-      if (!bRead)
-      {
-        DEBUG_ERROR_LOG(::GetLastError(), PLUGIN_ERROR_FILTER, PLUGIN_ERROR_FILTER_READ_FILE, "Filter::ParseFilters - Read")
-      }
-      else
-      {
-        isRead = true;
-
-        UINT dstSize = 0;
-        BYTE* buf = (BYTE*)fileContent.GetString();
-        UINT srcLength = fileContent.GetLength();
-        if (((encodingInfos[scores - 1].nCodePage) == 1200) || ((encodingInfos[scores - 1].nCodePage) == 1201))
-        {
-          srcLength = srcLength * 2;
-        }
-        hr = pMultiLanguage->ConvertString(NULL, encodingInfos[scores - 1].nCodePage, 1252, (BYTE*)buf, &srcLength, NULL, &dstSize);
-        char* bufferTmp = new char[dstSize + 1];
-        hr = pMultiLanguage->ConvertString(NULL, encodingInfos[scores - 1].nCodePage, 1252, (BYTE*)buf, &srcLength, (BYTE*)bufferTmp, &dstSize);
-        bufferTmp[dstSize] = 0;
-        //Unicode 
-        if ((encodingInfos[0].nCodePage == 1200) || (encodingInfos[0].nCodePage == 1201))
-        {
-          //remove BOM for Unicode
-          fileContent = (char*)bufferTmp;
-          fileContent.Delete(0, 1);
-
-        }
-        else 
-        {
-          wchar_t* fileContentBuffer = fileContent.GetBufferSetLength(dstSize);
-          memset(fileContentBuffer, 0, dstSize); 
-          memcpy(fileContentBuffer, bufferTmp, dstSize);
-        }
-        delete [] bufferTmp;
-
-      }
-
-      // Close file
-      ::CloseHandle(hFile);
-    }
-
-#ifdef PRODUCT_ADBLOCKPLUS
-  }
-#endif
-  if (isRead)
-  {
-    // Parse file string
-    int pos = 0;
-    CString filter = fileContent.Tokenize(L"\n\r", pos);
-
-    s_criticalSectionFilterMap.Lock();
-    {
-      while (pos >= 0)
-      {
-        // If the line is not commented out
-        if (!filter.Trim().IsEmpty() && filter.GetAt(0) != '!' && filter.GetAt(0) != '[')
-        {
-          int filterType = 0;
-
-          // We need to categorize the filters
-          // We have three options, whitelist, block or element hiding
-          // See http://adblockplus.org/en/filters for further documentation
-
-          // @@ indicates white listing rule
-          if (filter.Find(L"@@") == 0)
-          {
-            filterType = CFilter::filterTypeWhiteList;
-
-            filter.Delete(0, 2);
-          }
-          // If a filter contains ## then it is a element hiding rule
-          else if (filter.Find(L"#") >= 0)
-          {
-            filterType = CFilter::filterTypeElementHide;
-          } 
-          //Anything we do not support here
-          else if (filter.Find(L"*") == 0)
-          {
-            filterType = CFilter::filterTypeUnknown;
-          }
-          // Else, it is a general rule
-          else
-          {
-            filterType = CFilter::filterTypeBlocking;
-          }
-
-          try
-          {
-            // Element hiding not supported yet
-            if (filterType == CFilter::filterTypeElementHide)
-            { 
-              AddFilterElementHide(filter, filename);
-            }
-            else if (filterType != CFilter::filterTypeUnknown)
-            {
-              AddFilter(filter, filename, filterType);
-            }
-          }
-          catch(...)
-          {
-            //just ignore all errors we might get when adding filters
-          }
-        }
-
-        CString tmp = L"";
-        tmp.Format(L"Current %d, Total %d", pos, fileContent.GetLength());
-        filter = fileContent.Tokenize(L"\n\r", pos);
-        int nextPos = fileContent.Find(L"\n", pos + 1);
-        if (nextPos < 0)
-        {
-          filter = "";
-          pos = -1;
-
-        }
-        if ((filter.GetLength() > 5000) || (fileContent.GetLength() - 5 <= pos))
-        {
-          filter = "";
+          //just ignore all errors we might get when adding filters

[Felix Dahlke, 2013/04/03 05:35:27]

We should probably at least log exceptions.

         }
       }
     }
-    s_criticalSectionFilterMap.Unlock();
-  } 
+  }
+  s_criticalSectionFilterMap.Unlock();
 
   return isRead;
 }
 
-void CPluginFilter::ParseFilters(const TFilterFileList& list)
+void CPluginFilter::ClearFilters()
 {
   // Clear filter maps
   s_criticalSectionFilterMap.Lock();    
@@ -1469,29 +890,6 @@
     m_elementHideDomains.clear();
   }
   s_criticalSectionFilterMap.Unlock();
-
-  // Load the files
-#ifdef PRODUCT_ADBLOCKPLUS
-  CPluginClient* client = CPluginClient::GetInstance();
-#endif
-  for (TFilterFileList::const_iterator it = list.begin(); it != list.end(); ++it) 
-  {
-    ReadFilter(it->first, it->second);
-  }
-
-#ifdef PERSONAL_FILTER_FILE
-  ReadFilter(PERSONAL_FILTER_FILE);
-#endif
-
-#ifdef ENABLE_DEBUG_SELFTEST
-  CStringA sCount;
-  s_criticalSectionFilterMap.Lock();
-  {
-    sCount.Format("Block:%d/%d - BlockDef:%d - White:%d - WhiteDef:%d - Hide:%d/%d", m_filterMap[0][0].size(), m_filterMap[0][1].size(), m_filterMapDefault[0].size(), m_filterMap[1][0].size(), m_filterMapDefault[1].size(), m_elementHideTags.size() + m_elementHideTagsClass.size() + m_elementHideTagsId.size(), m_elementHideDomains.size());
-  }
-  s_criticalSectionFilterMap.Unlock();
-  DEBUG_GENERAL("*** Filter count:" + sCount);
-#endif
 }
 
 
@@ -1800,6 +1198,26 @@
     }
   }
 
+  CPluginClient* client = CPluginClient::GetInstance();
+  AdblockPlus::FilterEngine* filterEngine = client->GetFilterEngine();
+
+  src.OemToCharA();

[Wladimir Palant, 2013/04/03 12:42:19]

Are we really getting all URLs as ASCII? What about IDNs (http://россия.рф/), do
we get them as Punycode or are they completely broken?

[Felix Dahlke, 2013/04/03 12:51:04]

The problem is that ABP for IE uses UTF-16 internally (like most Windows
programs), but libadblockplus uses UTF-8. We talked about converting to/from
UTF-8 whenever we interface with libadblockplus, but I thought it's fine to just
support ASCII URLs for now and revisit this a bit later. Then again, it's
probably not a huge deal to get it right now.

[Oleksandr, 2013/04/03 13:12:17]

I've used CT2CA in the end (not included in this patchset). Wladimir it's
multibyte string, not really ASCII. So it supports UTF-8.

On 2013/04/03 12:51:04, Felix H. Dahlke wrote:

+
+  std::string contentTypeString = "";

[Wladimir Palant, 2013/04/03 12:42:19]

This needs a TODO comment: "Need to guess the content type".

+
+  std::string url = (char*)src.GetBuffer();

[Felix Dahlke, 2013/04/03 05:35:27]

I'm really leaving my comfort zone here, but some quick googling makes it look
like "(LPCSTR) src" is the canonical way to convert a CString to an std::string.

+  if (filterEngine->Matches(url, contentTypeString))
+  {
+    if (addDebug)
+    {
+      DEBUG_FILTER("Filter::ShouldBlock " + type + " YES")
+
+#ifdef ENABLE_DEBUG_RESULT
+        CPluginDebug::DebugResultBlocking(type, src);
+#endif
+    }
+    return true;
+  }
   const CFilter* blockFilter = MatchFilter(CFilter::filterTypeBlocking, src, contentType, domain);
   if (blockFilter)
   {
@@ -1817,7 +1235,7 @@
       DEBUG_FILTER("Filter::ShouldBlock " + type + " YES src:" + src + " - \"" + blockFilter->m_filterText + "\"")
 
 #ifdef ENABLE_DEBUG_RESULT
-        CPluginDebug::DebugResultBlocking(type, src, blockFilter->m_filterText, blockFilter->m_filterFile);
+        CPluginDebug::DebugResultBlocking(type, src);
 #endif
     }
   }
@@ -1980,88 +1398,3 @@
 
   return false;
 }
-
-#ifdef PRODUCT_ADBLOCKPLUS
-
-void CPluginFilter::CreateFilters()
-{
-  CPluginFilterLock lock("filter1.txt");
-  if (lock.IsLocked())
-  {
-    // Check file existence
-    std::ifstream is;
-    is.open(CPluginSettings::GetDataPath("filter1.txt"), std::ios_base::in);
-    if (is.is_open())
-    {
-      is.close();
-      return;
-    }
-
-    // Open file
-    HANDLE hFile = ::CreateFile(CPluginSettings::GetDataPath("filter1.txt"), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);  
-    if (hFile == INVALID_HANDLE_VALUE)
-    {
-      DEBUG_ERROR_LOG(::GetLastError(), PLUGIN_ERROR_FILTER, PLUGIN_ERROR_FILTER_CREATE_FILE_OPEN, "Filter::Create - CreateFile");
-    }
-    else
-    {
-      // Build filter string
-      CStringA line;
-      line += "[Adblock Plus 1.1]\r\n";
-      line += "! Checksum: 0xN4e5SegJGeZuYZQGhShQ\r\n";
-      line += "! EasyList - https://easylist.adblockplus.org/\r\n";
-      line += "! Last modified:  4 Nov 2010 16:30 UTC\r\n";
-      line += "! Expires: 5 days (update frequency)\r\n";
-      line += "! Licence: https://easylist-downloads.adblockplus.org/COPYING\r\n";
-      line += "!\r\n";
-      line += "! Please report any unblocked adverts or problems\r\n";
-      line += "! in the forums (http://forums.lanik.us/)\r\n";
-      line += "! or via e-mail (easylist.subscription@gmail.com).\r\n";
-      line += "!\r\n";
-      line += "!-----------------General advert blocking filters-----------------!\r\n";
-
-
-      // Write file
-      DWORD dwBytesWritten = 0;
-      if (::WriteFile(hFile, line.GetBuffer(), line.GetLength(), &dwBytesWritten, NULL) && dwBytesWritten == line.GetLength())
-      {
-        // Set correct version
-        CPluginSettings* settings = CPluginSettings::GetInstance();
-
-        settings->AddFilterUrl(CString(FILTERS_PROTOCOL) + CString(FILTERS_HOST) + "/easylist.txt", 1);
-        settings->AddFilterFileName(CString(FILTERS_PROTOCOL) + CString(FILTERS_HOST) + "/easylist.txt", "filter1.txt");
-        settings->Write();
-      }
-      else
-      {
-        DEBUG_ERROR_LOG(::GetLastError(), PLUGIN_ERROR_FILTER, PLUGIN_ERROR_FILTER_CREATE_FILE_WRITE, "Filter::Create - WriteFile");
-      }
-
-      // Close file
-      if (!::CloseHandle(hFile))
-      {
-        DEBUG_ERROR_LOG(::GetLastError(), PLUGIN_ERROR_FILTER, PLUGIN_ERROR_FILTER_CREATE_FILE_CLOSE, "Filter::Create - CloseHandle");
-      }
-    }
-  }
-}
-
-#endif // PRODUCT_ADBLOCKPLUS
-
-
-#ifdef PRODUCT_ADBLOCKPLUS
-
-bool CPluginFilter::IsAlive() const
-{
-  bool isAlive;
-
-  s_criticalSectionFilterMap.Lock();
-  {
-    isAlive = !m_filterMap[0][0].empty();
-  }
-  s_criticalSectionFilterMap.Unlock();
-
-  return isAlive;
-}
-
-#endif