Bug 1110446 P2 Cleanup stale caches/bodies if last session didn't shutdown cleanly. r=ehsan

dom/cache/DBSchema.cpp

@@ -537,6 +537,62 @@ IsCacheOrphaned(mozIStorageConnection* aConn, CacheId aCacheId,
   return rv;
 }
 
+nsresult
+FindOrphanedCacheIds(mozIStorageConnection* aConn,
+                     nsTArray<CacheId>& aOrphanedListOut)
+{
+  nsCOMPtr<mozIStorageStatement> state;
+  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT id FROM caches "
+    "WHERE id NOT IN (SELECT cache_id from storage);"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) {
+    CacheId cacheId = INVALID_CACHE_ID;
+    rv = state->GetInt64(0, &cacheId);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+    aOrphanedListOut.AppendElement(cacheId);
+  }
+
+  return rv;
+}
+
+nsresult
+GetKnownBodyIds(mozIStorageConnection* aConn, nsTArray<nsID>& aBodyIdListOut)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(aConn);
+
+  nsCOMPtr<mozIStorageStatement> state;
+  nsresult rv = aConn->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT request_body_id, response_body_id FROM entries;"
+  ), getter_AddRefs(state));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  bool hasMoreData = false;
+  while (NS_SUCCEEDED(state->ExecuteStep(&hasMoreData)) && hasMoreData) {
+    // extract 0 to 2 nsID structs per row
+    for (uint32_t i = 0; i < 2; ++i) {
+      bool isNull = false;
+
+      rv = state->GetIsNull(i, &isNull);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      if (!isNull) {
+        nsID id;
+        rv = ExtractId(state, i, &id);
+        if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+        aBodyIdListOut.AppendElement(id);
+      }
+    }
+  }
+
+  return rv;
+}
+
 nsresult
 CacheMatch(mozIStorageConnection* aConn, CacheId aCacheId,
            const CacheRequest& aRequest,

dom/cache/DBSchema.h

@@ -48,6 +48,13 @@ nsresult
 IsCacheOrphaned(mozIStorageConnection* aConn, CacheId aCacheId,
                 bool* aOrphanedOut);
 
+nsresult
+FindOrphanedCacheIds(mozIStorageConnection* aConn,
+                     nsTArray<CacheId>& aOrphanedListOut);
+
+nsresult
+GetKnownBodyIds(mozIStorageConnection* aConn, nsTArray<nsID>& aBodyIdListOut);
+
 nsresult
 CacheMatch(mozIStorageConnection* aConn, CacheId aCacheId,
            const CacheRequest& aRequest, const CacheQueryParams& aParams,

dom/cache/FileUtils.cpp

@@ -320,8 +320,108 @@ BodyIdToFile(nsIFile* aBaseDir, const nsID& aId, BodyFileType aType,
 } // anonymous namespace
 
 nsresult
-CreateMarkerFile(const QuotaInfo& aQuotaInfo)
+BodyDeleteOrphanedFiles(nsIFile* aBaseDir, nsTArray<nsID>& aKnownBodyIdList)
 {
+  MOZ_ASSERT(aBaseDir);
+
+  // body files are stored in a directory structure like:
+  //
+  //  /morgue/01/{01fdddb2-884d-4c3d-95ba-0c8062f6c325}.final
+  //  /morgue/02/{02fdddb2-884d-4c3d-95ba-0c8062f6c325}.tmp
+
+  nsCOMPtr<nsIFile> dir;
+  nsresult rv = aBaseDir->Clone(getter_AddRefs(dir));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Add the root morgue directory
+  rv = dir->Append(NS_LITERAL_STRING("morgue"));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  nsCOMPtr<nsISimpleEnumerator> entries;
+  rv = dir->GetDirectoryEntries(getter_AddRefs(entries));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+  // Iterate over all the intermediate morgue subdirs
+  bool hasMore = false;
+  while (NS_SUCCEEDED(rv = entries->HasMoreElements(&hasMore)) && hasMore) {
+    nsCOMPtr<nsISupports> entry;
+    rv = entries->GetNext(getter_AddRefs(entry));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    nsCOMPtr<nsIFile> subdir = do_QueryInterface(entry);
+
+    bool isDir = false;
+    rv = subdir->IsDirectory(&isDir);
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    // If a file got in here somehow, try to remove it and move on
+    if (NS_WARN_IF(!isDir)) {
+      rv = subdir->Remove(false /* recursive */);
+      MOZ_ASSERT(NS_SUCCEEDED(rv));
+      continue;
+    }
+
+    nsCOMPtr<nsISimpleEnumerator> subEntries;
+    rv = subdir->GetDirectoryEntries(getter_AddRefs(subEntries));
+    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+    // Now iterate over all the files in the subdir
+    bool subHasMore = false;
+    while(NS_SUCCEEDED(rv = subEntries->HasMoreElements(&subHasMore)) &&
+          subHasMore) {
+      nsCOMPtr<nsISupports> subEntry;
+      rv = subEntries->GetNext(getter_AddRefs(subEntry));
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      nsCOMPtr<nsIFile> file = do_QueryInterface(subEntry);
+
+      nsAutoCString leafName;
+      rv = file->GetNativeLeafName(leafName);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      // Delete all tmp files regardless of known bodies.  These are
+      // all considered orphans.
+      if (StringEndsWith(leafName, NS_LITERAL_CSTRING(".tmp"))) {
+        // remove recursively in case its somehow a directory
+        rv = file->Remove(true /* recursive */);
+        MOZ_ASSERT(NS_SUCCEEDED(rv));
+        continue;
+      }
+
+      nsCString suffix(NS_LITERAL_CSTRING(".final"));
+
+      // Otherwise, it must be a .final file.  If its not, then just
+      // skip it.
+      if (NS_WARN_IF(!StringEndsWith(leafName, suffix) ||
+                     leafName.Length() != NSID_LENGTH - 1 + suffix.Length())) {
+        continue;
+      }
+
+      // Finally, parse the uuid out of the name.  If its fails to parse,
+      // the ignore the file.
+      nsID id;
+      if (NS_WARN_IF(!id.Parse(leafName.BeginReading()))) {
+        continue;
+      }
+
+      if (!aKnownBodyIdList.Contains(id)) {
+        // remove recursively in case its somehow a directory
+        rv = file->Remove(true /* recursive */);
+        MOZ_ASSERT(NS_SUCCEEDED(rv));
+      }
+    }
+  }
+
+  return rv;
+}
+
+namespace {
+
+nsresult
+GetMarkerFileHandle(const QuotaInfo& aQuotaInfo, nsIFile** aFileOut)
+{
+  MOZ_ASSERT(aFileOut);
+
   nsCOMPtr<nsIFile> marker;
   nsresult rv = aQuotaInfo.mDir->Clone(getter_AddRefs(marker));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
@@ -332,6 +432,20 @@ CreateMarkerFile(const QuotaInfo& aQuotaInfo)
   rv = marker->Append(NS_LITERAL_STRING("context_open.marker"));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
+  marker.forget(aFileOut);
+
+  return rv;
+}
+
+} // anonymous namespace
+
+nsresult
+CreateMarkerFile(const QuotaInfo& aQuotaInfo)
+{
+  nsCOMPtr<nsIFile> marker;
+  nsresult rv = GetMarkerFileHandle(aQuotaInfo, getter_AddRefs(marker));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
   rv = marker->Create(nsIFile::NORMAL_FILE_TYPE, 0644);
   if (rv == NS_ERROR_FILE_ALREADY_EXISTS) {
     rv = NS_OK;
@@ -350,13 +464,7 @@ nsresult
 DeleteMarkerFile(const QuotaInfo& aQuotaInfo)
 {
   nsCOMPtr<nsIFile> marker;
-  nsresult rv = aQuotaInfo.mDir->Clone(getter_AddRefs(marker));
-  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
-
-  rv = marker->Append(NS_LITERAL_STRING("cache"));
-  if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
-
-  rv = marker->Append(NS_LITERAL_STRING("context_open.marker"));
+  nsresult rv = GetMarkerFileHandle(aQuotaInfo, getter_AddRefs(marker));
   if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
   rv = marker->Remove(/* recursive = */ false);
@@ -373,6 +481,20 @@ DeleteMarkerFile(const QuotaInfo& aQuotaInfo)
   return NS_OK;
 }
 
+bool
+MarkerFileExists(const QuotaInfo& aQuotaInfo)
+{
+  nsCOMPtr<nsIFile> marker;
+  nsresult rv = GetMarkerFileHandle(aQuotaInfo, getter_AddRefs(marker));
+  if (NS_WARN_IF(NS_FAILED(rv))) { return false; }
+
+  bool exists = false;
+  rv = marker->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) { return false; }
+
+  return exists;
+}
+
 } // namespace cache
 } // namespace dom
 } // namespace mozilla

dom/cache/FileUtils.h

@@ -50,12 +50,18 @@ BodyOpen(const QuotaInfo& aQuotaInfo, nsIFile* aBaseDir, const nsID& aId,
 nsresult
 BodyDeleteFiles(nsIFile* aBaseDir, const nsTArray<nsID>& aIdList);
 
+nsresult
+BodyDeleteOrphanedFiles(nsIFile* aBaseDir, nsTArray<nsID>& aKnownBodyIdList);
+
 nsresult
 CreateMarkerFile(const QuotaInfo& aQuotaInfo);
 
 nsresult
 DeleteMarkerFile(const QuotaInfo& aQuotaInfo);
 
+bool
+MarkerFileExists(const QuotaInfo& aQuotaInfo);
+
 } // namespace cache
 } // namespace dom
 } // namespace mozilla

dom/cache/Manager.cpp

@@ -30,15 +30,12 @@
 #include "nsThreadUtils.h"
 #include "nsTObserverArray.h"
 
-namespace {
 
-using mozilla::unused;
-using mozilla::dom::cache::Action;
-using mozilla::dom::cache::BodyCreateDir;
-using mozilla::dom::cache::BodyDeleteFiles;
-using mozilla::dom::cache::QuotaInfo;
-using mozilla::dom::cache::SyncDBAction;
-using mozilla::dom::cache::db::CreateSchema;
+namespace mozilla {
+namespace dom {
+namespace cache {
+
+namespace {
 
 // An Action that is executed when a Context is first created.  It ensures that
 // the directory and database are setup properly.  This lets other actions
@@ -54,23 +51,56 @@ public:
   RunSyncWithDBOnTarget(const QuotaInfo& aQuotaInfo, nsIFile* aDBDir,
                         mozIStorageConnection* aConn) override
   {
-    // TODO: init maintainance marker (bug 1110446)
-    // TODO: perform maintainance if necessary (bug 1110446)
-    // TODO: find orphaned caches in database (bug 1110446)
-    // TODO: have Context create/delete marker files in constructor/destructor
-    //       and only do expensive maintenance if that marker is present (bug 1110446)
-
     nsresult rv = BodyCreateDir(aDBDir);
     if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-    mozStorageTransaction trans(aConn, false,
-                                mozIStorageConnection::TRANSACTION_IMMEDIATE);
+    {
+      mozStorageTransaction trans(aConn, false,
+                                  mozIStorageConnection::TRANSACTION_IMMEDIATE);
 
-    rv = CreateSchema(aConn);
-    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+      rv = db::CreateSchema(aConn);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
 
-    rv = trans.Commit();
-    if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+      rv = trans.Commit();
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+    }
+
+    // If the Context marker file exists, then the last session was
+    // not cleanly shutdown.  In these cases sqlite will ensure that
+    // the database is valid, but we might still orphan data.  Both
+    // Cache objects and body files can be referenced by DOM objects
+    // after they are "removed" from their parent.  So we need to
+    // look and see if any of these late access objects have been
+    // orphaned.
+    //
+    // Note, this must be done after any schema version updates to
+    // ensure our DBSchema methods work correctly.
+    if (MarkerFileExists(aQuotaInfo)) {
+      NS_WARNING("Cache not shutdown cleanly! Cleaning up stale data...");
+      mozStorageTransaction trans(aConn, false,
+                                  mozIStorageConnection::TRANSACTION_IMMEDIATE);
+
+      // Clean up orphaned Cache objects
+      nsAutoTArray<CacheId, 8> orphanedCacheIdList;
+      nsresult rv = db::FindOrphanedCacheIds(aConn, orphanedCacheIdList);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+      for (uint32_t i = 0; i < orphanedCacheIdList.Length(); ++i) {
+        nsAutoTArray<nsID, 16> deletedBodyIdList;
+        rv = db::DeleteCacheId(aConn, orphanedCacheIdList[i], deletedBodyIdList);
+        if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+
+        rv = BodyDeleteFiles(aDBDir, deletedBodyIdList);
+        if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+      }
+
+      // Clean up orphaned body objects
+      nsAutoTArray<nsID, 64> knownBodyIdList;
+      rv = db::GetKnownBodyIds(aConn, knownBodyIdList);
+
+      rv = BodyDeleteOrphanedFiles(aDBDir, knownBodyIdList);
+      if (NS_WARN_IF(NS_FAILED(rv))) { return rv; }
+    }
 
     return rv;
   }
@@ -124,14 +154,6 @@ private:
   nsTArray<nsID> mDeletedBodyIdList;
 };
 
-} // anonymous namespace
-
-namespace mozilla {
-namespace dom {
-namespace cache {
-
-namespace {
-
 bool IsHeadRequest(CacheRequest aRequest, CacheQueryParams aParams)
 {
   return !aParams.ignoreMethod() && aRequest.method().LowerCaseEqualsLiteral("head");