Improvement of PIM-SQL Databases, but search queries are still limited.

Addressbook: Changed table layout. Now, we just need 1/3 of disk-space.
Todo: Started to add new attributes. Some type conversions missing.

3 files changed, 370 insertions, 45 deletions

diff --git a/libopie/pim/ocontactaccessbackend_sql.cpp b/libopie/pim/ocontactaccessbackend_sql.cpp
index 4afa5f3..132c9fc 100644
--- a/libopie/pim/ocontactaccessbackend_sql.cpp
+++ b/libopie/pim/ocontactaccessbackend_sql.cpp
@@ -16,2 +16,7 @@
  * $Log$
+ * Revision 1.2  2003/09/29 07:44:26  eilers
+ * Improvement of PIM-SQL Databases, but search queries are still limited.
+ * Addressbook: Changed table layout. Now, we just need 1/3 of disk-space.
+ * Todo: Started to add new attributes. Some type conversions missing.
+ *
  * Revision 1.1  2003/09/22 14:31:16  eilers
@@ -39,2 +44,18 @@
 
+
+
+
+// If defined, we use a horizontal table ( uid, attr1, attr2, attr3, ..., attrn ) instead
+// vertical like "uid, type, value".
+// DON'T DEACTIVATE THIS DEFINE IN PRODUCTIVE ENVIRONMENTS !!
+#define __STORE_HORIZONTAL_
+
+// Distinct loading is not very fast. If I expect that every person has just
+// one (and always one) 'Last Name', I can request all uid's for existing lastnames, 
+// which is faster..
+// But this may not be true for all entries, like company contacts..
+// The current AddressBook application handles this problem, but other may not.. (eilers)
+#define __USE_SUPERFAST_LOADQUERY
+
+
 /*
@@ -142,4 +163,3 @@ namespace {
         // 1. addressbook  : It contains General information about the contact (non custom)
-        // 2. dates        : Stuff like birthdays, anniversaries, etc.
-        // 3. custom_data  : Not official supported entries
+        // 2. custom_data  : Not official supported entries
         // All tables are connected by the uid of the contact.
@@ -150,2 +170,16 @@ namespace {
                 QString qu;
+#ifdef __STORE_HORIZONTAL_
+
+                qu += "create table addressbook( uid PRIMARY KEY ";
+
+                QStringList fieldList = OContactFields::untrfields( false );
+                for ( QStringList::Iterator it = ++fieldList.begin(); it != fieldList.end(); ++it ){
+                        qu += QString( ",\"%1\" VARCHAR(10)" ).arg( *it );
+                }
+                qu += " );";
+                
+                qu += "create table custom_data( uid INTEGER, id INTEGER, type VARCHAR, priority INTEGER, value VARCHAR, PRIMARY KEY /* identifier */ (uid, id) );";
+
+#else
+
                 qu += "create table addressbook( uid INTEGER, id INTEGER, type VARCHAR, priority INTEGER, value VARCHAR, PRIMARY KEY /* identifier */ (uid, id));";
@@ -153,2 +187,4 @@ namespace {
         //      qu += "create table dates( uid PRIMARY KEY, type, day, month, year, hour, minute, second );";
+
+#endif // __STORE_HORIZONTAL_
                 return qu;
@@ -162,3 +198,3 @@ namespace {
                 qu += "drop table custom_data;";
-                qu += "drop table dates;";
+        //      qu += "drop table dates;";
                 return qu;
@@ -166,8 +202,2 @@ namespace {
 
-// Distinct loading is not very fast. If I expect that every person has just
-// one (and always one) 'Last Name', I can request all uid's for existing lastnames, 
-// which is faster..
-// But this may not be true for all entries, like company contacts..
-// The current AddressBook application handles this problem, but other may not.. (eilers)
-#define __USE_SUPERFAST_LOADQUERY
 
@@ -177,7 +207,11 @@ namespace {
                 QString qu;
-#ifndef __USE_SUPERFAST_LOADQUERY 
-                qu += "select distinct uid from addressbook";
+#ifdef __STORE_HORIZONTAL_
+                qu += "select uid from addressbook";
 #else
+#  ifndef __USE_SUPERFAST_LOADQUERY 
+                qu += "select distinct uid from addressbook";
+#  else
                 qu += "select uid from addressbook where type = 'Last Name'";
-#endif
+#  endif // __USE_SUPERFAST_LOADQUERY 
+#endif // __STORE_HORIZONTAL_
                 
@@ -199,2 +233,55 @@ namespace {
 
+#ifdef __STORE_HORIZONTAL_
+                QString qu;
+                qu  += "insert into addressbook VALUES( " + 
+                        QString::number( m_contact.uid() );
+
+                // Get all information out of the contact-class
+                // Remember: The category is stored in contactMap, too ! 
+                QMap<int, QString> contactMap = m_contact.toMap();
+                
+                QStringList fieldList = OContactFields::untrfields( false );
+                QMap<QString, int> translate = OContactFields::untrFieldsToId();
+                for ( QStringList::Iterator it = ++fieldList.begin(); it != fieldList.end(); ++it ){
+                        // Convert Column-String to Id and get value for this id..
+                        // Hmmm.. Maybe not very cute solution..
+                        int id = translate[*it];
+                        switch ( id ){
+                        case Qtopia::Birthday:{
+                                // These entries should stored in a special format
+                                // year-month-day
+                                QDate day = m_contact.birthday();
+                                if ( day.isValid() ){
+                                        qu += QString(",\"%1-%2-%3\"")
+                                                .arg( day.year() )
+                                                .arg( day.month() )
+                                                .arg( day.day() );
+                                } else {
+                                        qu += ",\"\"";
+                                }
+                        }
+                                break;
+                        case Qtopia::Anniversary:{
+                                // These entries should stored in a special format
+                                // year-month-day
+                                QDate day = m_contact.anniversary();
+                                if ( day.isValid() ){
+                                        qu += QString(",\"%1-%2-%3\"")
+                                                .arg( day.year() )
+                                                .arg( day.month() )
+                                                .arg( day.day() );
+                                } else {
+                                        qu += ",\"\"";
+                                }
+                        }
+                                break;
+
+                        default:
+                                qu += QString( ",\"%1\"" ).arg( contactMap[id] );
+                        }
+                }
+                qu += " );";
+
+                        
+#else
                 // Get all information out of the contact-class
@@ -202,3 +289,2 @@ namespace {
                 QMap<int, QString> contactMap = m_contact.toMap();
-                QMap<QString, QString> customMap = m_contact.toExtraMap();
                 
@@ -263,4 +349,9 @@ namespace {
 
+        #endif          //__STORE_HORIZONTAL_
                 // Now add custom data..
+#ifdef __STORE_HORIZONTAL_
+                int id = 0;
+#endif
                 id = 0;
+                QMap<QString, QString> customMap = m_contact.toExtraMap();
                 for( QMap<QString, QString>::Iterator it = customMap.begin(); 
@@ -279,3 +370,2 @@ namespace {
                         }       
-                
                 // qu  += "commit;";
@@ -292,4 +382,2 @@ namespace {
                         + QString::number(m_uid) + ";";
-                qu += "DELETE from dates where uid = " 
-                        + QString::number(m_uid) + ";";
                 qu += "DELETE from custom_data where uid = " 
@@ -327,2 +415,11 @@ namespace {
         */
+#ifdef __STORE_HORIZONTAL_
+        QString FindQuery::single()const{
+                QString qu = "select *";
+                qu += " from addressbook where uid = " + QString::number(m_uid);
+
+                // qWarning("find query: %s", qu.latin1() );
+                return qu;
+        }
+#else
         QString FindQuery::single()const{
@@ -332,2 +429,3 @@ namespace {
         }
+#endif
 
@@ -377,3 +475,3 @@ OContactAccessBackend_SQL::OContactAccessBackend_SQL ( const QString& /* appname
 
-        qWarning("C'tor OContactAccessBackend_SQL ends: %d", t.elapsed() );
+        qWarning("C'tor OContactAccessBackend_SQL ends: %d ms", t.elapsed() );
 }
@@ -480,3 +578,3 @@ OContact OContactAccessBackend_SQL::find ( int uid ) const
 
-        qWarning("OContactAccessBackend_SQL::find() needed: %d", t.elapsed() );
+        qWarning("OContactAccessBackend_SQL::find() needed: %d ms", t.elapsed() );
         return retContact;
@@ -488,4 +586,47 @@ QArray<int> OContactAccessBackend_SQL::queryByExample ( const OContact &query, i
 {
-        QArray<int> nix(0);
-        return nix;
+        QString qu = "SELECT uid FROM addressbook WHERE";
+
+        QMap<int, QString> queryFields = query.toMap();
+        QStringList fieldList = OContactFields::untrfields( false );
+        QMap<QString, int> translate = OContactFields::untrFieldsToId();
+
+        // Convert every filled field to a SQL-Query
+        bool isAnyFieldSelected = false;
+        for ( QStringList::Iterator it = ++fieldList.begin(); it != fieldList.end(); ++it ){
+                int id = translate[*it];
+                QString queryStr = queryFields[id];
+                if ( !queryStr.isEmpty() ){
+                        isAnyFieldSelected = true;
+                        switch( id ){
+                        default:
+                                // Switching between case sensitive and insensitive...
+                                // LIKE is not case sensitive, GLOB is case sensitive
+                                // Do exist a better solution to switch this ?
+                                if ( settings & OContactAccess::IgnoreCase )
+                                        qu += "(\"" + *it + "\"" + " LIKE " + "'" 
+                                                + queryStr.replace(QRegExp("\\*"),"%") + "'" + ") AND "; 
+                                else
+                                        qu += "(\"" + *it + "\"" + " GLOB " + "'" 
+                                                + queryStr + "'" + ") AND "; 
+                                        
+                        }
+                }
+        }
+        // Skip trailing "AND"
+        if ( isAnyFieldSelected )
+                qu = qu.left( qu.length() - 4 );
+
+        qWarning( "queryByExample query: %s", qu.latin1() );
+
+        // Execute query and return the received uid's
+        OSQLRawQuery raw( qu );
+        OSQLResult res = m_driver->query( &raw );
+        if ( res.state() != OSQLResult::Success ){
+                QArray<int> empty;
+                return empty;
+        }
+
+        QArray<int> list = extractUids( res );
+
+                return list;    
 }
@@ -500,3 +641,4 @@ const uint OContactAccessBackend_SQL::querySettings()
 {
-        return 0;
+        return OContactAccess::IgnoreCase 
+                || OContactAccess::WildCards;
 }
@@ -505,3 +647,46 @@ bool OContactAccessBackend_SQL::hasQuerySettings (uint querySettings) const
 {
-        return false;
+        /* OContactAccess::IgnoreCase, DateDiff, DateYear, DateMonth, DateDay
+         * may be added with any of the other settings. IgnoreCase should never used alone.
+         * Wildcards, RegExp, ExactMatch should never used at the same time...
+         */
+
+        // Step 1: Check whether the given settings are supported by this backend
+        if ( ( querySettings & ( 
+                                OContactAccess::IgnoreCase
+                                | OContactAccess::WildCards
+        //                      | OContactAccess::DateDiff
+        //                      | OContactAccess::DateYear
+        //                      | OContactAccess::DateMonth
+        //                      | OContactAccess::DateDay
+        //                      | OContactAccess::RegExp
+        //                      | OContactAccess::ExactMatch
+                               ) ) != querySettings )
+                return false;
+
+        // Step 2: Check whether the given combinations are ok..
+
+        // IngoreCase alone is invalid
+        if ( querySettings == OContactAccess::IgnoreCase )
+                return false;
+
+        // WildCards, RegExp and ExactMatch should never used at the same time 
+        switch ( querySettings & ~( OContactAccess::IgnoreCase
+                                    | OContactAccess::DateDiff
+                                    | OContactAccess::DateYear
+                                    | OContactAccess::DateMonth
+                                    | OContactAccess::DateDay
+                                    )
+                 ){
+        case OContactAccess::RegExp:
+                return ( true );
+        case OContactAccess::WildCards:
+                return ( true );
+        case OContactAccess::ExactMatch:
+                return ( true );
+        case 0: // one of the upper removed bits were set..
+                return ( true );
+        default:
+                return ( false );
+        }
+
 }
@@ -513,6 +698,10 @@ QArray<int> OContactAccessBackend_SQL::sorted( bool asc,  int , int ,  int )
 
-        // Not implemented..
+#ifdef __STORE_HORIZONTAL_
+        QString query = "SELECT uid FROM addressbook ";
+        query += "ORDER BY \"Last Name\" ";
+#else
         QString query = "SELECT uid FROM addressbook WHERE type = 'Last Name' ";
+        query += "ORDER BY upper( value )";
+#endif
 
-        query += "ORDER BY value ";
         if ( !asc )
@@ -520,3 +709,3 @@ QArray<int> OContactAccessBackend_SQL::sorted( bool asc,  int , int ,  int )
 
-        qWarning("sorted query is: %s", query.latin1() ); 
+        // qWarning("sorted query is: %s", query.latin1() ); 
 
@@ -554,3 +743,3 @@ void OContactAccessBackend_SQL::update()
 
-        qWarning("Update ends %d", t.elapsed() );
+        qWarning("Update ends %d ms", t.elapsed() );
 }
@@ -578,2 +767,68 @@ QArray<int> OContactAccessBackend_SQL::extractUids( OSQLResult& res ) const
 
+#ifdef __STORE_HORIZONTAL_
+QMap<int, QString>  OContactAccessBackend_SQL::requestNonCustom( int uid ) const
+{
+        QTime t;
+        t.start();
+
+        QMap<int, QString> nonCustomMap;
+        
+        int t2needed = 0;
+        int t3needed = 0;
+        QTime t2;
+        t2.start();
+        FindQuery query( uid );
+        OSQLResult res_noncustom = m_driver->query( &query );
+        t2needed = t2.elapsed();
+
+        OSQLResultItem resItem = res_noncustom.first();
+
+        QTime t3;
+        t3.start();
+        // Now loop through all columns
+        QStringList fieldList = OContactFields::untrfields( false );
+        QMap<QString, int> translate = OContactFields::untrFieldsToId();
+        for ( QStringList::Iterator it = ++fieldList.begin(); it != fieldList.end(); ++it ){
+                // Get data for the selected column and store it with the
+                // corresponding id into the map..
+
+                int id =  translate[*it];
+                QString value = resItem.data( (*it) );
+                
+                // qWarning("Reading %s... found: %s", (*it).latin1(), value.latin1() );
+
+                switch( id ){
+                case Qtopia::Birthday:
+                case Qtopia::Anniversary:{
+                        // Birthday and Anniversary are encoded special ( yyyy-mm-dd )
+                        QStringList list = QStringList::split( '-', value );
+                        QStringList::Iterator lit = list.begin();
+                        int year  = (*lit).toInt();
+                        int month = (*(++lit)).toInt();
+                        int day   = (*(++lit)).toInt();
+                        if ( ( day != 0 ) && ( month != 0 ) && ( year != 0 ) ){
+                             QDate date( year, month, day );
+                             nonCustomMap.insert( id, OConversion::dateToString( date ) );
+                        }
+                }
+                        break;
+                case Qtopia::AddressCategory:
+                        qWarning("Category is: %s", value.latin1() );
+                default:
+                        nonCustomMap.insert( id, value );
+                }
+        }
+
+        // First insert uid
+        nonCustomMap.insert( Qtopia::AddressUid, resItem.data( "uid" ) ); 
+        t3needed = t3.elapsed();
+
+        // qWarning("Adding UID: %s", resItem.data( "uid" ).latin1() );
+        qWarning("RequestNonCustom needed: insg.:%d ms, query: %d ms, mapping: %d ms", 
+                 t.elapsed(), t2needed, t3needed  );
+
+        return nonCustomMap;
+}
+#else
+
 QMap<int, QString>  OContactAccessBackend_SQL::requestNonCustom( int uid ) const
@@ -635,3 +890,3 @@ QMap<int, QString>  OContactAccessBackend_SQL::requestNonCustom( int uid ) const
 
-        qWarning("RequestNonCustom needed: ins:%d, query: %d, mapping: %d", t.elapsed(), t2needed, t3needed  );
+        qWarning("RequestNonCustom needed: insg.:%d ms, query: %d ms, mapping: %d ms", t.elapsed(), t2needed, t3needed  );
         return nonCustomMap;
@@ -639,2 +894,4 @@ QMap<int, QString>  OContactAccessBackend_SQL::requestNonCustom( int uid ) const
 
+#endif // __STORE_HORIZONTAL_
+
 QMap<QString, QString>  OContactAccessBackend_SQL::requestCustom( int uid ) const
@@ -661,3 +918,3 @@ QMap<QString, QString>  OContactAccessBackend_SQL::requestCustom( int uid ) cons
 
-        qWarning("RequestCustom needed: %d", t.elapsed() );
+        qWarning("RequestCustom needed: %d ms", t.elapsed() );
         return customMap;
diff --git a/libopie/pim/ocontactfields.cpp b/libopie/pim/ocontactfields.cpp
index 831a596..7206f0d 100644
--- a/libopie/pim/ocontactfields.cpp
+++ b/libopie/pim/ocontactfields.cpp
@@ -175,2 +175,5 @@ QStringList OContactFields::untrfields( bool sorted )
 
+    list.append( mapIdToStr[ Qtopia::AddressUid ] );
+    list.append( mapIdToStr[ Qtopia::AddressCategory ] );
+
     list.append( mapIdToStr[ Qtopia::Title ] );
@@ -213,2 +216,5 @@ QMap<int, QString> OContactFields::idToTrFields()
 
+        ret_map.insert( Qtopia::AddressUid, QObject::tr( "User Id" ) );
+        ret_map.insert( Qtopia::AddressCategory, QObject::tr( "Categories" ) );
+
         ret_map.insert( Qtopia::Title, QObject::tr( "Name Title") );
@@ -276,2 +282,5 @@ QMap<int, QString> OContactFields::idToUntrFields()
 
+        ret_map.insert( Qtopia::AddressUid, "User Id" );
+        ret_map.insert( Qtopia::AddressCategory, "Categories" );
+
         ret_map.insert( Qtopia::Title, "Name Title" );
@@ -330,2 +339,3 @@ QMap<int, QString> OContactFields::idToUntrFields()
         ret_map.insert( Qtopia::Notes, "Notes" );
+        ret_map.insert( Qtopia::Groups, "Groups" );
 
diff --git a/libopie/pim/otodoaccesssql.cpp b/libopie/pim/otodoaccesssql.cpp
index 23e0c3e..d255c66 100644
--- a/libopie/pim/otodoaccesssql.cpp
+++ b/libopie/pim/otodoaccesssql.cpp
@@ -120,4 +120,5 @@ namespace {
         QString qu;
-        qu += "create table todolist( uid, categories, completed, progress, ";
-        qu += "summary, DueDate, priority, description )";
+        qu += "create table todolist( uid PRIMARY KEY, categories, completed, ";
+        qu += "description, summary, priority, DueDate, progress ,  state, ";
+        qu += "Recurrence, notifiers, maintainer, startdate, completeddate)";
         return qu;
@@ -129,3 +130,5 @@ namespace {
         QString qu;
-        qu += "select distinct uid from todolist";
+        // We do not need "distinct" here. The primary key is always unique..
+        //qu += "select distinct uid from todolist";
+        qu += "select uid from todolist";
 
@@ -152,8 +155,42 @@ namespace {
             day = date.day();
-        }
+        }
+        int sYear = 0, sMonth = 0, sDay = 0;
+        if( m_todo.hasStartDate() ){
+                QDate sDate = m_todo.startDate();
+                sYear = sDate.year();
+                sMonth= sDate.month();
+                sDay  = sDate.day();
+        } 
+
+        int eYear = 0, eMonth = 0, eDay = 0;
+        if( m_todo.hasCompletedDate() ){
+                QDate eDate = m_todo.completedDate();
+                eYear = eDate.year();
+                eMonth= eDate.month();
+                eDay  = eDate.day();
+        } 
         QString qu;
-        qu  = "insert into todolist VALUES(" +  QString::number( m_todo.uid() ) + ",'" + m_todo.idsToString( m_todo.categories() ) + "',";
-        qu += QString::number( m_todo.isCompleted() ) + "," + QString::number( m_todo.progress() ) + ",";
-        qu += "'"+m_todo.summary()+"','"+QString::number(year)+"-"+QString::number(month)+"-"+QString::number(day)+"',";
-        qu += QString::number(m_todo.priority() ) +",'" + m_todo.description() + "')";
+        qu  = "insert into todolist VALUES(" 
+                +  QString::number( m_todo.uid() ) + "," 
+                + "'" + m_todo.idsToString( m_todo.categories() ) + "'" + ","
+                +       QString::number( m_todo.isCompleted() )         + ","  
+                + "'" + m_todo.description()                      + "'" + ","
+                + "'" + m_todo.summary()                          + "'" + "," 
+                +       QString::number(m_todo.priority() )             + ","
+                + "'" + QString::number(year) + "-" 
+                +       QString::number(month)
+                +       "-" + QString::number( day )              + "'" + ","
+                +       QString::number( m_todo.progress() )            + ","
+                + "''"                                                  + "," // state (conversion needed)
+        //      +       QString::number( m_todo.state() )               + ","
+                + "''"                                                  + "," // Recurrence (conversion needed)
+                + "''"                                                  + "," // Notifiers (conversion needed)
+                + "''"                                                  + "," // Maintainers (conversion needed)
+                + "'" + QString::number(sYear) + "-" 
+                      + QString::number(sMonth)
+                +       "-" + QString::number(sDay)               + "'" + ","
+                + "'" + QString::number(eYear) + "-"
+                +       QString::number(eMonth)
+                +       "-"+QString::number(eDay)                 + "'"
+                + ")";
 
@@ -194,4 +231,3 @@ namespace {
     QString FindQuery::single()const{
-        QString qu = "select uid, categories, completed, progress, summary, ";
-        qu += "DueDate, priority, description from todolist where uid = " + QString::number(m_uid);
+        QString qu = "select * from todolist where uid = " + QString::number(m_uid);
         return qu;
@@ -199,4 +235,3 @@ namespace {
     QString FindQuery::multi()const {
-        QString qu = "select uid, categories, completed, progress, summary, ";
-        qu += "DueDate, priority, description from todolist where ";
+        QString qu = "select * from todolist where ";
         for (uint i = 0; i < m_uids.count(); i++ ) {
@@ -290,3 +325,3 @@ OTodo OTodoAccessBackendSQL::find( int uid, const QArray<int>& ints,
                                    uint cur, Frontend::CacheDirection dir ) const{
-    int CACHE = readAhead();
+    uint CACHE = readAhead();
     qWarning("searching for %d", uid );
@@ -474,4 +509,4 @@ OTodo OTodoAccessBackendSQL::todo( OSQLResultItem& item )const {
     qWarning("todo");
-    bool has = false; QDate da = QDate::currentDate();
-    has = date( da, item.data("DueDate") );
+    bool hasDueDate = false; QDate dueDate = QDate::currentDate();
+    hasDueDate = date( dueDate, item.data("DueDate") );
     QStringList cats = QStringList::split(";", item.data("categories") );
@@ -480,4 +515,20 @@ OTodo OTodoAccessBackendSQL::todo( OSQLResultItem& item )const {
               cats, item.data("summary"), item.data("description"),
-              item.data("progress").toUShort(), has, da,
+              item.data("progress").toUShort(), hasDueDate, dueDate,
               item.data("uid").toInt() );
+
+    bool isOk;
+    int prioInt = QString( item.data("priority") ).toInt( &isOk );
+    if ( isOk )
+            to.setPriority( prioInt );
+
+    bool hasStartDate = false; QDate startDate = QDate::currentDate();
+    hasStartDate = date( startDate, item.data("startdate") );
+    bool hasCompletedDate = false; QDate completedDate = QDate::currentDate();
+    hasCompletedDate = date( completedDate, item.data("completeddate") );
+
+    if ( hasStartDate )
+            to.setStartDate( startDate );
+    if ( hasCompletedDate )
+            to.setCompletedDate( completedDate );
+
     return to;
@@ -572,3 +623,10 @@ QBitArray OTodoAccessBackendSQL::supports()const {
 
-        static QBitArray ar = sup();
+        QBitArray ar( OTodo::CompletedDate + 1 );
+        ar.fill( true );
+        ar[OTodo::CrossReference] = false;
+        ar[OTodo::State ] = false;
+        ar[OTodo::Reminders] = false;
+        ar[OTodo::Notifiers] = false;
+        ar[OTodo::Maintainer] = false;
+
         return ar;