From e904db041ba06082f659b094d2335c9300106555 Mon Sep 17 00:00:00 2001 From: Gilles Filippini Date: Mon, 14 Mar 2022 19:46:55 +0100 Subject: [PATCH] Initial import --- LICENSE | 1 + import_history_mms_sms | 252 +++++++++++++++++++++++++++++++++++++++++ save_files | 13 +++ to_sqlite | 66 +++++++++++ 4 files changed, 332 insertions(+) create mode 100644 LICENSE create mode 100755 import_history_mms_sms create mode 100755 save_files create mode 100755 to_sqlite diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d5d21ff --- /dev/null +++ b/LICENSE @@ -0,0 +1 @@ +Public Domain diff --git a/import_history_mms_sms b/import_history_mms_sms new file mode 100755 index 0000000..70ef51c --- /dev/null +++ b/import_history_mms_sms @@ -0,0 +1,252 @@ +#!/bin/bash +set -euo pipefail + +database="$1" +history_db="$HOME/.purple/chatty/db/chatty-history.db" + +CHATTY_ID_PHONE_VALUE=1 +MMS_SMS_USERID=1 +PROTOCOL_MMS_SMS=1 +THREAD_DIRECT_CHAT=0 +THREAD_GROUP_CHAT=1 +THREAD_VISIBILITY_VISIBLE=0 +CHATTY_DIRECTION_IN=1 +CHATTY_DIRECTION_OUT=2 +MESSAGE_TYPE_IMAGE=9 +MESSAGE_TYPE_TEXT=1 +MESSAGE_TYPE_MMS=12 +MESSAGE_STATUS_DELIVERED=4 +MESSAGE_STATUS_DELIVERY_FAILED=7 +MESSAGE_STATUS_READ=5 +CHATTY_FILE_DOWNLOADED=1 + +function join_by { local IFS="$1"; shift; echo "$@"; } + +# This must be adapted after your country code +country_code=+33 +country_regex='^0([0-9]{9})$' +function normalize_address () { + address="$1" + if [[ "$address" =~ $country_regex ]]; then + address="$country_code${BASH_REMATCH[1]}" + fi + echo "$address" +} + +function add_user () { + address="$1" + alias="$2" + value_address="'$address'" + value_alias="$([ -z "$alias" ] && echo null || echo "'$alias'")" + userid="$(sqlite3 "$history_db" "insert or ignore into users(username, type, alias) VALUES ( + $value_address, + $CHATTY_ID_PHONE_VALUE, + $value_alias) + ON CONFLICT(username,type) + DO UPDATE SET alias=coalesce($value_alias,alias); + select id from users where username=$value_address and type=$CHATTY_ID_PHONE_VALUE;")" + echo "$userid" +} + +declare -A aliases + +# Import SMS +while read msgdata; do + echo "$msgdata" + IFS='|' read -a msgdata_array <<<"$msgdata" + msgid="${msgdata_array[0]}" + address="$(normalize_address "${msgdata_array[1]}")" + display_name="${msgdata_array[2]}" + date="${msgdata_array[3]}" + direction="${msgdata_array[4]}" + msg_status="${msgdata_array[5]}" + aliases["${address}"]="$display_name" + if [ "$direction" = "$$CHATTY_DIRECTION_OUT" ]; then + if [ "$msg_status" = 0 ]; then + msg_status="$MESSAGE_STATUS_DELIVERED" + else + msg_status="$MESSAGE_STATUS_DELIVERY_FAILED" + fi + else + msg_status="$MESSAGE_STATUS_READ" + fi + sender_id="$(add_user "$address" "${aliases["$address"]}")" + thread_name="'$address'" + thread_alias="'${aliases["$address"]}'" + if [ "$thread_alias" = "''" ]; then + thread_alias=null + fi + local_user_accountid="$(sqlite3 "$history_db" "SELECT id from accounts where user_id=$MMS_SMS_USERID and protocol=$PROTOCOL_MMS_SMS;")" + threadid="$(sqlite3 "$history_db" "INSERT INTO threads(name,alias,account_id,type,visibility) VALUES ( + $thread_name, + $thread_alias, + $local_user_accountid, + $THREAD_DIRECT_CHAT, + $THREAD_VISIBILITY_VISIBLE) + ON CONFLICT(name,account_id,type) + DO UPDATE SET alias=$thread_alias, visibility=$THREAD_VISIBILITY_VISIBLE; + select id from threads where name=$thread_name and account_id=$local_user_accountid and type=$THREAD_DIRECT_CHAT;")" + sqlite3 "$history_db" "INSERT OR IGNORE INTO thread_members(thread_id, user_id) VALUES ( + $threadid, + $sender_id);" + # Retreive text message + body="$(sqlite3 "$database" "SELECT body FROM sms where _id=$msgid;" | sed "s/'/''/g")" + body="'${body:-}'" + body_type="$MESSAGE_TYPE_TEXT" + # Create message record + uid="'$(sqlite3 "$history_db" "select substr(u,1,8)||'-'||substr(u,9,4)||'-4'||substr(u,13,3)||'-'||v||substr(u,17,3)||'-'||substr(u,21,12) + from (select lower(hex(randomblob(16))) as u, substr('89ab',abs(random()) % 4 + 1, 1) as v)")'" + messageid="$(sqlite3 "$history_db" "INSERT INTO messages(uid,thread_id,sender_id,body,body_type,direction,time,preview_id,status) VALUES( + $uid, + $threadid, + $sender_id, + $body, + $body_type, + $(( -(direction*2-3) )), + $date, + null, + $msg_status) + ON CONFLICT (uid,thread_id,body,time) + DO UPDATE SET status=$msg_status; + select id from messages where uid=$uid;")" +done <<<"$(sqlite3 "$database" "select _id, address, display_name, date/1000, type, status from sms order by cast(_id as integer);")" + +# import MMS +while read msgdata; do + echo "$msgdata" + IFS='|' read -a msgdata_array <<<"$msgdata" + msgid="${msgdata_array[0]}" + uid="'${msgdata_array[1]}'" + date="${msgdata_array[2]}" + subject="${msgdata_array[3]:-}" + if [ -n "$subject" ]; then + subject="'$subject'" + else + subject=null + fi + IFS='|' read -a sender <<<$(sqlite3 "$database" "select address, display_name from sender where msg_id='$msgid';") + address="$(normalize_address "${sender[0]}")" + display_name="${sender[1]:-}" + users=() + if [ -z "$address" ]; then + direction=$CHATTY_DIRECTION_OUT + sender_id=null + msg_status="$MESSAGE_STATUS_DELIVERED" + while read line; do + IFS='|' read -a recipient <<<"$line" + users+=("$(normalize_address "${recipient[0]}")") + aliases["${recipient[0]}"]="${recipient[1]:-}" + done <<<$(sqlite3 "$database" "select address, display_name from recipient where msg_id='$msgid';") + else + # We don't collect recipient addresses for inbound messages + # They have little value, especially when you receive MMS sent to a whole address book + direction=$CHATTY_DIRECTION_IN + msg_status="$MESSAGE_STATUS_READ" + users=("$address") + aliases["$address"]="$display_name" + fi + thread_aliases=() + thread_members=() + for user in "${users[@]}"; do + thread_members+=("$(add_user "$user" "${aliases["$user"]:-}")") + thread_aliases+=("${aliases["$user"]:-$user}") + done + sender_id="${sender_id:-${thread_members[0]}}" + thread_name="'$(join_by "," "${users[@]}")'" + thread_alias="'$(join_by "," "${thread_aliases[@]}")'" + if [ "$thread_alias" = "$thread_name" ]; then + thread_alias=null + fi + local_user_accountid="$(sqlite3 "$history_db" "SELECT id from accounts where user_id=$MMS_SMS_USERID and protocol=$PROTOCOL_MMS_SMS;")" + if [ "${#users[@]}" -gt 1 ]; then + thread_type="$THREAD_GROUP_CHAT" + else + thread_type="$THREAD_DIRECT_CHAT" + fi + threadid="$(sqlite3 "$history_db" "INSERT INTO threads(name,alias,account_id,type,visibility) VALUES ( + $thread_name, + $thread_alias, + $local_user_accountid, + $thread_type, + $THREAD_VISIBILITY_VISIBLE) + ON CONFLICT(name,account_id,type) + DO UPDATE SET alias=$thread_alias, visibility=$THREAD_VISIBILITY_VISIBLE; + select id from threads where name=$thread_name and account_id=$local_user_accountid and type=$thread_type;")" + for member in "${thread_members[@]}"; do + sqlite3 "$history_db" "INSERT OR IGNORE INTO thread_members(thread_id, user_id) VALUES ( + $threadid, + $member);" + done + # Concat plain text parts as the body + body="$(sqlite3 "$database" "SELECT text FROM part where mid=$msgid and ct='text/plain' order by cast(_id as integer);" | sed "s/'/''/g")" + body="'${body:-}'" + body_type="$MESSAGE_TYPE_MMS" + # Create message record + messageid="$(sqlite3 "$history_db" "INSERT INTO messages(uid,thread_id,sender_id,body,body_type,direction,time,preview_id,status,subject) VALUES( + $uid, + $threadid, + $sender_id, + $body, + $body_type, + $(( -(direction*2-3) )), + $date, + null, + $msg_status, + $subject) + ON CONFLICT (uid,thread_id,body,time) + DO UPDATE SET status=$msg_status; + select id from messages where uid=$uid;")" + # Process binary attachements + parent="${XDG_DATA_HOME:-$HOME/.local/share}/chatty" + attachments_info="$(sqlite3 "$database" "SELECT ct, savepath FROM part where mid=$msgid and lower(ct) not in ('application/smil', 'text/plain') order by cast(_id as integer);")" + [ -z "$attachments_info" ] || while read line; do + IFS='|' read -a attachment <<<"$line" + mime_type="${attachment[0]}" + savepath="${attachment[1]}" + name="'$(basename "$savepath")'" + url="'file://$(urlencode -m "$savepath")'" + path="'${savepath#$parent/}'" + size="$(stat --printf="%s" "$savepath")" + mime_type_id="$(sqlite3 "$history_db" "INSERT OR IGNORE INTO mime_type(name) VALUES ('$mime_type'); + select id from mime_type where name='$mime_type';")" + # File record + fileid="$(sqlite3 "$history_db" "INSERT INTO files(name,url,path,mime_type_id,size,status) VALUES ( + $name, + $url, + $path, + $mime_type_id, + $size, + $CHATTY_FILE_DOWNLOADED) + ON CONFLICT(url) + DO UPDATE SET path=$path, size=$size, status=$CHATTY_FILE_DOWNLOADED; + select id from files where url=$url;")" + # File_metadata + if [[ "$mime_type" =~ ^(image|video|audio)/ ]]; then + raw_metadata="$( (ffmpeg -i "$savepath" 2<&1 || true) | grep -E '(Duration|Stream.* Video):')" + if [[ "$mime_type" =~ ^(image|video)/ ]]; then + if [[ "$raw_metadata" =~ \ ([0-9]+)x([0-9]+)\ ]]; then + width="${BASH_REMATCH[1]}" + height="${BASH_REMATCH[2]}" + fi + fi + if [[ "$mime_type" =~ ^(video|audio)/ ]]; then + if [[ "$raw_metadata" =~ Duration:\ ([0-9]+):([0-9]+):([0-9]+)(\.([0-9]+))?, ]]; then + duration=$((${BASH_REMATCH[1]}*3600+${BASH_REMATCH[2]}*60+${BASH_REMATCH[2]})) + fi + fi + if [ -n "${width:-}${height:-}${duration:-}" ]; then + width="${width:-null}" + height="${height:-null}" + duration="${duration:-null}" + sqlite3 "$history_db" "INSERT OR IGNORE INTO file_metadata(file_id,width,height,duration) VALUES ( + $fileid, + $width, + $height, + $duration);" + fi + fi + sqlite3 "$history_db" "INSERT OR IGNORE INTO message_files(message_id,file_id) VALUES ( + $messageid, + $fileid);" + done <<<"$attachments_info" +done <<<"$(sqlite3 "$database" "select _id, m_id, date, sub from mms order by cast(mms._id as integer);")" diff --git a/save_files b/save_files new file mode 100755 index 0000000..8ddb8aa --- /dev/null +++ b/save_files @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +database="$1" + +# Saving binary files only +# .smil parts are ignored (not used by Chatty) +# Plain text attachments will be converted into message bodies +# cf. chatty:src/mm/chatty-mmsd.c:chatty_mmsd_process_mms_message_attachments +while read path; do + mkdir -p "$(dirname "$path")" + sqlite3 "$database" "select binary_data from part where savepath='$path';" | base64 -d >"$path" +done <<<"$(sqlite3 "$database" "select savepath from part where binary_data is not null;")" diff --git a/to_sqlite b/to_sqlite new file mode 100755 index 0000000..b93ad21 --- /dev/null +++ b/to_sqlite @@ -0,0 +1,66 @@ +#!/bin/bash +set -euo pipefail + +wkdir= +debug= +function cleanup () { + [ -z "$wkdir" -a -z "$debug" ] || rm -fr "$wkdir" +} +trap 'cleanup' EXIT +if [ "$1" = "-d" ]; then + debug=1 + shift +fi +input_file=$(readlink -f "$1") +wkdir=$(mktemp -d) +cd "$wkdir" + +# Generate one JSON file per table into the temporary directory +jq '[map(select(has("recipient_addresses")|not))[]]' "$input_file" >sms.json +jq '[map(select(has("recipient_addresses")))[] | del(.parts, .recipient_addresses, .sender_address)]' "$input_file" >mms.json +jq '[map(select(has("recipient_addresses")))[].sender_address]' "$input_file" >sender.json +jq '[map(select(has("recipient_addresses")))[].recipient_addresses[]]' "$input_file" >recipient.json +jq '[map(select(has("parts")))[].parts[]]' "$input_file" >part.json + +# Tables creation +sms_keys=$(jq -r '[.[] | keys[]] | unique[]' sms.json) +mms_keys=$(jq -r '[.[] | keys[]] | unique[]' mms.json) +address_keys=$(jq -r '[.[] | keys[]] | unique[]' recipient.json) +part_keys=$(jq -r '[.[] | keys[]] | unique[]' part.json) + +function create_table () { + name="$1" + shift + echo -n "CREATE TABLE $name(" + echo -n $(echo "$@" | sed 's/ / TEXT,/g;s/$/ TEXT/') + echo ");" +} + +create_table sms $sms_keys +create_table mms $mms_keys +create_table sender $address_keys +create_table recipient $address_keys +create_table part $part_keys savepath + +# Populate +jq -r '.[] | to_entries | "INSERT INTO sms (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' sms.json +jq -r '.[] | to_entries | "INSERT INTO mms (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' mms.json +jq -r '.[] | to_entries | "INSERT INTO sender (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' sender.json +jq -r '.[] | to_entries | "INSERT INTO recipient (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' recipient.json +jq -r '.[] | to_entries | "INSERT INTO part (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' part.json + +# Fix MMS out sender +echo "UPDATE sender set address=null where address in ('insert-address-token', 'Unknown address');" + +# Delete MMS with no part +echo "DELETE FROM mms where not exists (SELECT 1 from part where part.mid=mms._id);" + +# Build savepath for each part +# Part: +# date = mms.date +# address = sender.address || moi +# uid = cl ? cl : (cid sans les '<>') +# tag = date+address+uid +# savepath = $user_data_dir/chatty/mms/tag +user_data_dir="${XDG_DATA_HOME:-$HOME/.local/share}/chatty/mms" +echo "UPDATE part set savepath=(select '$user_data_dir/' || mms.date || (CASE WHEN sender.address IS NOT null THEN sender.address ELSE '' END) || mms.m_id || '/' || replace(replace(replace((CASE WHEN part.cl IS NOT null THEN part.cl ELSE trim(part.cid,'<>') END),'(','_'),')','_'),' ','_') from mms, sender where mms._id=part.mid and sender.msg_id=part.mid);"