Flatbuffers serialization, aggregation optimization, build improvements

.gitignore

@@ -1,32 +0,0 @@
-# Prerequisites
-*.d
-
-# Compiled Object files
-*.slo
-*.lo
-*.o
-*.obj
-
-# Precompiled Headers
-*.gch
-*.pch
-
-# Compiled Dynamic libraries
-*.so
-*.dylib
-*.dll
-
-# Fortran module files
-*.mod
-*.smod
-
-# Compiled Static libraries
-*.lai
-*.la
-*.a
-*.lib
-
-# Executables
-*.exe
-*.out
-*.app

common/encryption/encrypt.h

@@ -1,44 +1,18 @@
 #ifndef ENCRYPT_H_
 #define ENCRYPT_H_
 
-#define CIPHER_KEY_SIZE 16
-#define CIPHER_IV_SIZE  12
-#define CIPHER_TAG_SIZE 16
-#define SHA_DIGEST_SIZE 32
-#define CIPHER_PK_SIZE 512
-#define SIG_ALLOC_SIZE 1024
-
-#include <iostream> 
-#include <map>
-#include <vector>
-#include <string>
-#include <string.h>
-#include <stdio.h>
-#include "serialization.h"
 #include "crypto.h"
 
-#include "mbedtls/config.h"
-#include "mbedtls/gcm.h"
-#include "mbedtls/entropy.h"    // mbedtls_entropy_context
-#include "mbedtls/ctr_drbg.h"   // mbedtls_ctr_drbg_context
-#include "mbedtls/cipher.h"     // MBEDTLS_CIPHER_ID_AES
-#include "mbedtls/gcm.h"        // mbedtls_gcm_context
-#include "mbedtls/pk.h"
-#include "mbedtls/rsa.h"
-#include "mbedtls/sha256.h"
-#include "mbedtls/x509_crt.h"
-#include "mbedtls/error.h"
-
-void encrypt_bytes(unsigned char* model_data, size_t data_len, unsigned char** ciphertext) {
-
+void encrypt_bytes(uint8_t* model_data, size_t data_len, uint8_t** ciphertext) {
     mbedtls_gcm_context gcm;
     mbedtls_gcm_init(&gcm);
 
-    unsigned char key[] = "abcdefghijklmnop";
+    // FIXME: hardcoded key
+    uint8_t key[] = "abcdefghijklmnop";
 
-    unsigned char* output = new unsigned char[data_len * sizeof(unsigned char)];
-    unsigned char* iv = new unsigned char[CIPHER_IV_SIZE * sizeof(unsigned char)];
-    unsigned char* tag = new unsigned char[CIPHER_TAG_SIZE * sizeof(unsigned char)];
+    uint8_t* output = ciphertext[0];
+    uint8_t* iv = ciphertext[1];
+    uint8_t* tag = ciphertext[2];
 
     int ret = encrypt_symm(
         key,
@@ -50,19 +24,14 @@ void encrypt_bytes(unsigned char* model_data, size_t data_len, unsigned char** c
         iv,
         tag
     );
-
-    *ciphertext = output;
-    *(ciphertext + 1) = iv;
-    *(ciphertext + 2) = tag;
 }
 
-void decrypt_bytes(unsigned char* model_data, unsigned char* iv, unsigned char* tag, size_t data_len, unsigned char** text) {
-
+void decrypt_bytes(uint8_t* model_data, uint8_t* iv, uint8_t* tag, size_t data_len, uint8_t** text) {
     mbedtls_gcm_context gcm;
     mbedtls_gcm_init(&gcm);
 
-    unsigned char key[] = "abcdefghijklmnop";
-    unsigned char* out = new unsigned char[data_len];
+    // FIXME: hardcoded key
+    uint8_t key[] = "abcdefghijklmnop";
 
     decrypt_symm(
         key,

common/encryption/serialization.h

@@ -1,73 +1,54 @@
 #ifndef SERIALIZATION_H_
 #define SERIALIZATION_H_
 
-#include <iostream> 
 #include <map>
 #include <vector>
-#include <string>
-#include <string.h>
-#include <stdio.h>
+#include "flatbuffers/model_generated.h"
 
-std::string serialize(std::map<std::string, std::vector<double>> model) {
-    // Serialize string:float[] map into SSSS:[VVVVVVVV]
-
-    std::vector<unsigned char> serialized;
+uint8_t* serialize(std::map<std::string, std::vector<double>> model,
+                            int* serialized_buffer_size) {
+    flatbuffers::FlatBufferBuilder builder;
+    std::vector<flatbuffers::Offset<secagg::KVPair>> features;
 
     for (const auto &[name, values]: model) {
-        for (char const &c: name) {
-            serialized.push_back(c);
-        }
-        serialized.push_back(':');
-        serialized.push_back('[');
-        for (const double &val: values) {
-            std::string val_string = std::to_string(val);
-            for (char const &c: val_string) {
-                serialized.push_back(c);
-            }
-            serialized.push_back(',');
-        }
-        serialized.push_back(']');
-        serialized.push_back('/');
+        auto key = builder.CreateString(name);
+        auto value = builder.CreateVector(values);
+        auto kvpair = secagg::CreateKVPair(builder, key, value);
+        features.push_back(kvpair);
     }
+    auto model_features = builder.CreateVector(features);
+    auto model_offset = secagg::CreateModel(builder, model_features);
+    builder.Finish(model_offset);
 
-    std::string s(serialized.begin(), serialized.end());
-    return s;
-}
-
-std::map<std::string, std::vector<double>> deserialize(std::string serialized_str) {
-    // Splits each entry on '/'
-    std::vector<std::string> entries; 
-    std::string delimitEntries = "/";
-    size_t pos = 0;
+    uint8_t* model_buffer = builder.GetBufferPointer();
+    int model_buffer_size = builder.GetSize();
 
-    while ((pos = serialized_str.find(delimitEntries)) != std::string::npos) {
-        std:: string token = serialized_str.substr(0, pos);
-        entries.push_back(token);
-        serialized_str.erase(0, pos+delimitEntries.length());
-    }
+    uint8_t* ret_buffer = new uint8_t[model_buffer_size];
+    memcpy(ret_buffer, model_buffer, sizeof(uint8_t) * model_buffer_size);
+    *serialized_buffer_size = model_buffer_size;
+    return ret_buffer;
+}
 
-    // Recreate the original map entries
+std::map<std::string, std::vector<double>> deserialize(uint8_t* serialized_buffer) {
     std::map<std::string, std::vector<double>> demodel;
-    for (const std::string &entry: entries) {
-        std::string delimiter = ":";
-        std::string key = entry.substr(0, entry.find(delimiter));
-        std::string values = entry.substr(entry.find(delimiter), entry.length());
-        // values.length()-3 to remove trailing characters at the end (:[ and ])
-        values = values.substr(2, values.length()-3);
-        std::vector<double> newValues;
-        pos = 0;
-        std::string delimitValues = ",";
-        while ((pos = values.find(",")) != std::string::npos) {
-            std::string value = values.substr(0, pos);
-            double valueDouble = ::atof(value.c_str());
-            newValues.push_back(valueDouble);
-            values.erase(0, pos+delimitValues.length());
+
+    auto model = secagg::GetModel(serialized_buffer);
+    auto kvpairs = model->kv();
+    auto num_kvs = kvpairs->size();
+    for (int i = 0; i < num_kvs; i++) {
+        std::vector<double> feature_values;
+        auto pair = kvpairs->Get(i);
+
+        // Key is a string
+        auto key = pair->key()->str();
+        auto value = pair->value();
+        for (int j = 0; j < value->size(); j++) {
+            auto feature_value = value->Get(j);
+            feature_values.push_back(feature_value);
         }
-        demodel.insert({key, newValues});
+        demodel.insert({key, feature_values});
     }
-
     return demodel;
-    // END DESERIALIZATION
 
 }
 

common/flatbuffers/model.fbs

@@ -0,0 +1,13 @@
+// Example IDL file for our monster's schema.
+namespace secagg;
+
+table KVPair {
+  key:string;
+  value:[double];
+}
+
+table Model {
+  kv:[KVPair];
+}
+
+root_type Model;