/* This Source Code Form is subject to the terms of the Mozilla Public *License,v.2.0.IfacopyoftheMPLwasnotdistributedwiththis
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
usecrate::{
client::CollectionMetadata, client::CollectionSignature,
schema::RemoteSettingsConnectionInitializer, Attachment, RemoteSettingsRecord, Result,
}; use camino::Utf8PathBuf; use rusqlite::{params, Connection, OpenFlags, OptionalExtension, Transaction}; use serde_json; use sha2::{Digest, Sha256};
use sql_support::{open_database::open_database_with_flags, ConnExt};
/// Internal storage type /// /// This will store downloaded records/attachments in a SQLite database. Nothing is implemented /// yet other than the initial API. /// /// Most methods input a `collection_url` parameter, is a URL that includes the remote settings /// server, bucket, and collection. If the `collection_url` for a get method does not match the one /// for a set method, then this means the application has switched their remote settings config and /// [Storage] should pretend like nothing is stored in the database. /// /// The reason for this is the [crate::RemoteSettingsService::update_config] method. If a consumer /// passes a new server or bucket to `update_config`, we don't want to be using cached data from /// the previous config. pubstruct Storage {
conn: Connection,
}
/// Get the last modified timestamp for the stored records /// /// Returns None if no records are stored or if `collection_url` does not match the /// last `collection_url` passed to `insert_collection_content` pubfn get_last_modified_timestamp(&self, collection_url: &str) -> Result<Option<u64>> { letmut stmt = self
.conn
.prepare("SELECT last_modified FROM collection_metadata WHERE collection_url = ?")?; let result: Option<u64> = stmt
.query_row((collection_url,), |row| row.get(0))
.optional()?;
Ok(result)
}
/// Get cached records for this collection /// /// Returns None if no records are stored or if `collection_url` does not match the `collection_url` passed /// to `insert_collection_content`. pubfn get_records(
&mutself,
collection_url: &str,
) -> Result<Option<Vec<RemoteSettingsRecord>>> { let tx = self.conn.transaction()?;
let fetched = tx.exists( "SELECT 1 FROM collection_metadata WHERE collection_url = ?",
(collection_url,),
)?; let result = if fetched { // If fetched before, get the records from the records table let records: Vec<RemoteSettingsRecord> = tx
.prepare("SELECT data FROM records WHERE collection_url = ?")?
.query_map(params![collection_url], |row| row.get::<_, Vec<u8>>(0))?
.map(|data| serde_json::from_slice(&data.unwrap()).unwrap())
.collect();
Ok(Some(records))
} else {
Ok(None)
};
tx.commit()?;
result
}
/// Get cached metadata for this collection /// /// Returns None if no data is stored or if `collection_url` does not match the `collection_url` passed /// to `insert_collection_content`. pubfn get_collection_metadata(
&self,
collection_url: &str,
) -> Result<Option<CollectionMetadata>> { letmut stmt_metadata = self.conn.prepare( "SELECT bucket, signature, x5u FROM collection_metadata WHERE collection_url = ?",
)?;
/// Get cached attachment data /// /// This returns the last attachment data sent to [Self::set_attachment]. /// /// Returns None if no attachment data is stored or if `collection_url` does not match the `collection_url` /// passed to `set_attachment`. pubfn get_attachment(
&self,
collection_url: &str,
metadata: Attachment,
) -> Result<Option<Vec<u8>>> { letmut stmt = self
.conn
.prepare("SELECT data FROM attachments WHERE id = ? AND collection_url = ?")?;
iflet Some(data) = stmt
.query_row((metadata.location, collection_url), |row| {
row.get::<_, Vec<u8>>(0)
})
.optional()?
{ // Return None if data doesn't match expected metadata if data.len() as u64 != metadata.size { return Ok(None);
} let hash = format!("{:x}", Sha256::digest(&data)); if hash != metadata.hash { return Ok(None);
}
Ok(Some(data))
} else {
Ok(None)
}
}
/// Set cached content for this collection. pubfn insert_collection_content(
&mutself,
collection_url: &str,
records: &[RemoteSettingsRecord],
last_modified: u64,
metadata: CollectionMetadata,
) -> Result<()> { let tx = self.conn.transaction()?;
// Delete ALL existing records and metadata for with different collection_urls. // // This way, if a user (probably QA) switches the remote settings server in the middle of a // browser sessions, we'll delete the stale data from the previous server.
tx.execute( "DELETE FROM records where collection_url <> ?",
[collection_url],
)?;
tx.execute( "DELETE FROM collection_metadata where collection_url <> ?",
[collection_url],
)?;
/// Insert/remove/update rows in the records table based on a records list /// /// Returns the max last modified record from the list fn update_record_rows(
tx: &Transaction<'_>,
collection_url: &str,
records: &[RemoteSettingsRecord],
) -> Result<u64> { // Find the max last_modified time while inserting records letmut max_last_modified = 0;
{ letmut insert_stmt = tx.prepare( "INSERT OR REPLACE INTO records (id, collection_url, data) VALUES (?, ?, ?)",
)?; letmut delete_stmt = tx.prepare("DELETE FROM records WHERE id=?")?; for record in records { if record.deleted {
delete_stmt.execute(params![&record.id])?;
} else {
max_last_modified = max_last_modified.max(record.last_modified); let data = serde_json::to_vec(&record)?;
insert_stmt.execute(params![record.id, collection_url, data])?;
}
}
}
Ok(max_last_modified)
}
/// Update the collection metadata after setting/merging records fn update_collection_metadata(
tx: &Transaction<'_>,
collection_url: &str,
last_modified: u64,
metadata: CollectionMetadata,
) -> Result<()> { // Update the metadata
tx.execute( "INSERT OR REPLACE INTO collection_metadata \
(collection_url, last_modified, bucket, signature, x5u) \
VALUES (?, ?, ?, ?, ?)",
(
collection_url,
last_modified,
metadata.bucket,
metadata.signature.signature,
metadata.signature.x5u,
),
)?;
Ok(())
}
/// Set the attachment data stored in the database, clearing out any previously stored data pubfn set_attachment(
&mutself,
collection_url: &str,
location: &str,
attachment: &[u8],
) -> Result<()> { let tx = self.conn.transaction()?;
// Delete ALL existing attachments for every collection_url
tx.execute( "DELETE FROM attachments WHERE collection_url != ?",
params![collection_url],
)?;
tx.execute( "INSERT OR REPLACE INTO ATTACHMENTS \
(id, collection_url, data) \
VALUES (?, ?, ?)",
params![location, collection_url, attachment,],
)?;
tx.commit()?;
Ok(())
}
/// Empty out all cached values and start from scratch. This is called when /// RemoteSettingsService::update_config() is called, since that could change the remote /// settings server which would invalidate all cached data. pubfn empty(&mutself) -> Result<()> { let tx = self.conn.transaction()?;
tx.execute("DELETE FROM records", [])?;
tx.execute("DELETE FROM attachments", [])?;
tx.execute("DELETE FROM collection_metadata", [])?;
tx.commit()?;
Ok(())
}
}
#[cfg(test)] mod tests { usesuper::Storage; usecrate::{
client::CollectionMetadata, client::CollectionSignature, Attachment, RemoteSettingsRecord,
Result, RsJsonObject,
}; use sha2::{Digest, Sha256};
// Set records
storage.insert_collection_content(
collection_url,
&records, 300,
CollectionMetadata::default(),
)?;
// Get records let fetched_records = storage.get_records(collection_url)?;
assert!(fetched_records.is_some()); let fetched_records = fetched_records.unwrap();
assert_eq!(fetched_records.len(), 2);
assert_eq!(fetched_records, records);
// Get records when none are set let fetched_records = storage.get_records(collection_url)?;
assert!(fetched_records.is_none());
// Get last modified timestamp when no records let last_modified = storage.get_last_modified_timestamp(collection_url)?;
assert!(last_modified.is_none());
// Set empty records
storage.insert_collection_content(
collection_url,
&Vec::<RemoteSettingsRecord>::default(), 42,
CollectionMetadata::default(),
)?;
// Get records let fetched_records = storage.get_records(collection_url)?;
assert_eq!(fetched_records, Some(Vec::new()));
// Get last modified timestamp when no records let last_modified = storage.get_last_modified_timestamp(collection_url)?;
assert_eq!(last_modified, Some(42));
let attachment = &[0x18, 0x64]; let collection_url = "https://example.com/api"; let attachment_metadata = Attachment {
filename: "abc".to_string(),
mimetype: "application/json".to_string(),
location: "tmp".to_string(),
hash: format!("{:x}", Sha256::digest(attachment)),
size: attachment.len() as u64,
};
// Store attachment
storage.set_attachment(collection_url, &attachment_metadata.location, attachment)?;
// Get attachment let fetched_attachment = storage.get_attachment(collection_url, attachment_metadata)?;
assert!(fetched_attachment.is_some()); let fetched_attachment = fetched_attachment.unwrap();
assert_eq!(fetched_attachment, attachment);
let attachment_1 = &[0x18, 0x64]; let attachment_2 = &[0x12, 0x48];
let attachment_metadata_1 = Attachment {
filename: "abc".to_string(),
mimetype: "application/json".to_string(),
location: "tmp".to_string(),
hash: format!("{:x}", Sha256::digest(attachment_1)),
size: attachment_1.len() as u64,
};
let attachment_metadata_2 = Attachment {
filename: "def".to_string(),
mimetype: "application/json".to_string(),
location: "tmp".to_string(),
hash: format!("{:x}", Sha256::digest(attachment_2)),
size: attachment_2.len() as u64,
};
// Store first attachment
storage.set_attachment(
collection_url,
&attachment_metadata_1.location,
attachment_1,
)?;
// Replace attachment with new data
storage.set_attachment(
collection_url,
&attachment_metadata_2.location,
attachment_2,
)?;
// Get attachment let fetched_attachment = storage.get_attachment(collection_url, attachment_metadata_2)?;
assert!(fetched_attachment.is_some()); let fetched_attachment = fetched_attachment.unwrap();
assert_eq!(fetched_attachment, attachment_2);
let attachment_1 = &[0x18, 0x64]; let attachment_2 = &[0x12, 0x48];
let attachment_metadata_1 = Attachment {
filename: "abc".to_string(),
mimetype: "application/json".to_string(),
location: "first_tmp".to_string(),
hash: format!("{:x}", Sha256::digest(attachment_1)),
size: attachment_1.len() as u64,
};
let attachment_metadata_2 = Attachment {
filename: "def".to_string(),
mimetype: "application/json".to_string(),
location: "second_tmp".to_string(),
hash: format!("{:x}", Sha256::digest(attachment_2)),
size: attachment_2.len() as u64,
};
// Set attachments for two different collections
storage.set_attachment(
collection_url_1,
&attachment_metadata_1.location,
attachment_1,
)?;
storage.set_attachment(
collection_url_2,
&attachment_metadata_2.location,
attachment_2,
)?;
// Verify that only the attachment for the second collection remains let fetched_attachment_1 =
storage.get_attachment(collection_url_1, attachment_metadata_1)?;
assert!(fetched_attachment_1.is_none());
let fetched_attachment_2 =
storage.get_attachment(collection_url_2, attachment_metadata_2)?;
assert!(fetched_attachment_2.is_some()); let fetched_attachment_2 = fetched_attachment_2.unwrap();
assert_eq!(fetched_attachment_2, attachment_2);
Ok(())
}
#[test] fn test_storage_get_attachment_not_found() -> Result<()> { let storage = Storage::new(":memory:".into())?;
// Get attachment that doesn't exist let fetched_attachment = storage.get_attachment(collection_url, metadata)?;
assert!(fetched_attachment.is_none());
let metadata = records[1]
.clone()
.attachment
.expect("No attachment metadata for record");
// Set records and attachment
storage.insert_collection_content(
collection_url,
&records, 42,
CollectionMetadata::default(),
)?;
storage.set_attachment(collection_url, &metadata.location, attachment)?;
// Verify they are stored let fetched_records = storage.get_records(collection_url)?;
assert!(fetched_records.is_some()); let fetched_attachment = storage.get_attachment(collection_url, metadata.clone())?;
assert!(fetched_attachment.is_some());
// Empty the storage
storage.empty()?;
// Verify they are deleted let fetched_records = storage.get_records(collection_url)?;
assert!(fetched_records.is_none()); let fetched_attachment = storage.get_attachment(collection_url, metadata)?;
assert!(fetched_attachment.is_none());
// Set records for collection_url1
storage.insert_collection_content(
collection_url1,
&records_collection_url1, 42,
CollectionMetadata::default(),
)?; // Verify records for collection_url1 let fetched_records = storage.get_records(collection_url1)?;
assert!(fetched_records.is_some()); let fetched_records = fetched_records.unwrap();
assert_eq!(fetched_records.len(), 1);
assert_eq!(fetched_records, records_collection_url1);
// Set records for collection_url2, which will clear records for all collections
storage.insert_collection_content(
collection_url2,
&records_collection_url2, 300,
CollectionMetadata::default(),
)?;
// Verify that records for collection_url1 have been cleared let fetched_records = storage.get_records(collection_url1)?;
assert!(fetched_records.is_none());
// Verify records for collection_url2 are correctly stored let fetched_records = storage.get_records(collection_url2)?;
assert!(fetched_records.is_some()); let fetched_records = fetched_records.unwrap();
assert_eq!(fetched_records.len(), 1);
assert_eq!(fetched_records, records_collection_url2);
// Verify last modified timestamps only for collection_url2 let last_modified1 = storage.get_last_modified_timestamp(collection_url1)?;
assert_eq!(last_modified1, None); let last_modified2 = storage.get_last_modified_timestamp(collection_url2)?;
assert_eq!(last_modified2, Some(300));
// Verify updated records let fetched_records = storage.get_records(collection_url)?;
assert!(fetched_records.is_some());
assert_eq!(fetched_records.unwrap(), updated_records);
// Verify last modified timestamp let last_modified = storage.get_last_modified_timestamp(collection_url)?;
assert_eq!(last_modified, Some(300));
Ok(())
}
// Quick way to generate the fields data for our mock records fn test_fields(data: &str) -> RsJsonObject { letmut map = serde_json::Map::new();
map.insert("data".into(), data.into());
map
}
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.