Initial import
This commit is contained in:
Gilles Filippini
commit
6fc4adcec5
5 changed files with 349 additions and 0 deletions
1
LICENSE
Normal file
1
LICENSE
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
Public Domain
|
||||||
17
README.md
Normal file
17
README.md
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
These scripts are a POC to demonstrate how to import Android SMS/MMS history
|
||||||
|
into Chatty.
|
||||||
|
|
||||||
|
Here is how it works:
|
||||||
|
|
||||||
|
1. Export SMS/MMS from the Android device using [SMS Import / Export](https://github.com/tmo1/sms-ie)
|
||||||
|
2. Convert the generated JSON file into a SQLite database:
|
||||||
|
|
||||||
|
`$ ./to_sqlite <file.json> | sqlite3 <smsmms.db>`
|
||||||
|
|
||||||
|
3. Save the MMS attachements
|
||||||
|
$ ./save_files <smsmms.db>
|
||||||
|
4. Import into a *pre-existing* Chatty history database (You must have
|
||||||
|
launched Chatty at least once):
|
||||||
|
$ ./import_history_mms_sms <smsmms.db>
|
||||||
|
|
||||||
|
Et voilà!
|
||||||
252
import_history_mms_sms
Executable file
252
import_history_mms_sms
Executable file
|
|
@ -0,0 +1,252 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
database="$1"
|
||||||
|
history_db="$HOME/.purple/chatty/db/chatty-history.db"
|
||||||
|
|
||||||
|
CHATTY_ID_PHONE_VALUE=1
|
||||||
|
MMS_SMS_USERID=1
|
||||||
|
PROTOCOL_MMS_SMS=1
|
||||||
|
THREAD_DIRECT_CHAT=0
|
||||||
|
THREAD_GROUP_CHAT=1
|
||||||
|
THREAD_VISIBILITY_VISIBLE=0
|
||||||
|
CHATTY_DIRECTION_IN=1
|
||||||
|
CHATTY_DIRECTION_OUT=2
|
||||||
|
MESSAGE_TYPE_IMAGE=9
|
||||||
|
MESSAGE_TYPE_TEXT=1
|
||||||
|
MESSAGE_TYPE_MMS=12
|
||||||
|
MESSAGE_STATUS_DELIVERED=4
|
||||||
|
MESSAGE_STATUS_DELIVERY_FAILED=7
|
||||||
|
MESSAGE_STATUS_READ=5
|
||||||
|
CHATTY_FILE_DOWNLOADED=1
|
||||||
|
|
||||||
|
function join_by { local IFS="$1"; shift; echo "$@"; }
|
||||||
|
|
||||||
|
# This must be adapted after your country code
|
||||||
|
country_code=+33
|
||||||
|
country_regex='^0([0-9]{9})$'
|
||||||
|
function normalize_address () {
|
||||||
|
address="$1"
|
||||||
|
if [[ "$address" =~ $country_regex ]]; then
|
||||||
|
address="$country_code${BASH_REMATCH[1]}"
|
||||||
|
fi
|
||||||
|
echo "$address"
|
||||||
|
}
|
||||||
|
|
||||||
|
function add_user () {
|
||||||
|
address="$1"
|
||||||
|
alias="$2"
|
||||||
|
value_address="'$address'"
|
||||||
|
value_alias="$([ -z "$alias" ] && echo null || echo "'$alias'")"
|
||||||
|
userid="$(sqlite3 "$history_db" "insert or ignore into users(username, type, alias) VALUES (
|
||||||
|
$value_address,
|
||||||
|
$CHATTY_ID_PHONE_VALUE,
|
||||||
|
$value_alias)
|
||||||
|
ON CONFLICT(username,type)
|
||||||
|
DO UPDATE SET alias=coalesce($value_alias,alias);
|
||||||
|
select id from users where username=$value_address and type=$CHATTY_ID_PHONE_VALUE;")"
|
||||||
|
echo "$userid"
|
||||||
|
}
|
||||||
|
|
||||||
|
declare -A aliases
|
||||||
|
|
||||||
|
# Import SMS
|
||||||
|
while read msgdata; do
|
||||||
|
echo "$msgdata"
|
||||||
|
IFS='|' read -a msgdata_array <<<"$msgdata"
|
||||||
|
msgid="${msgdata_array[0]}"
|
||||||
|
address="$(normalize_address "${msgdata_array[1]}")"
|
||||||
|
display_name="${msgdata_array[2]}"
|
||||||
|
date="${msgdata_array[3]}"
|
||||||
|
direction="${msgdata_array[4]}"
|
||||||
|
msg_status="${msgdata_array[5]}"
|
||||||
|
aliases["${address}"]="$display_name"
|
||||||
|
if [ "$direction" = "$$CHATTY_DIRECTION_OUT" ]; then
|
||||||
|
if [ "$msg_status" = 0 ]; then
|
||||||
|
msg_status="$MESSAGE_STATUS_DELIVERED"
|
||||||
|
else
|
||||||
|
msg_status="$MESSAGE_STATUS_DELIVERY_FAILED"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
msg_status="$MESSAGE_STATUS_READ"
|
||||||
|
fi
|
||||||
|
sender_id="$(add_user "$address" "${aliases["$address"]}")"
|
||||||
|
thread_name="'$address'"
|
||||||
|
thread_alias="'${aliases["$address"]}'"
|
||||||
|
if [ "$thread_alias" = "''" ]; then
|
||||||
|
thread_alias=null
|
||||||
|
fi
|
||||||
|
local_user_accountid="$(sqlite3 "$history_db" "SELECT id from accounts where user_id=$MMS_SMS_USERID and protocol=$PROTOCOL_MMS_SMS;")"
|
||||||
|
threadid="$(sqlite3 "$history_db" "INSERT INTO threads(name,alias,account_id,type,visibility) VALUES (
|
||||||
|
$thread_name,
|
||||||
|
$thread_alias,
|
||||||
|
$local_user_accountid,
|
||||||
|
$THREAD_DIRECT_CHAT,
|
||||||
|
$THREAD_VISIBILITY_VISIBLE)
|
||||||
|
ON CONFLICT(name,account_id,type)
|
||||||
|
DO UPDATE SET alias=$thread_alias, visibility=$THREAD_VISIBILITY_VISIBLE;
|
||||||
|
select id from threads where name=$thread_name and account_id=$local_user_accountid and type=$THREAD_DIRECT_CHAT;")"
|
||||||
|
sqlite3 "$history_db" "INSERT OR IGNORE INTO thread_members(thread_id, user_id) VALUES (
|
||||||
|
$threadid,
|
||||||
|
$sender_id);"
|
||||||
|
# Retreive text message
|
||||||
|
body="$(sqlite3 "$database" "SELECT body FROM sms where _id=$msgid;" | sed "s/'/''/g")"
|
||||||
|
body="'${body:-}'"
|
||||||
|
body_type="$MESSAGE_TYPE_TEXT"
|
||||||
|
# Create message record
|
||||||
|
uid="'$(sqlite3 "$history_db" "select substr(u,1,8)||'-'||substr(u,9,4)||'-4'||substr(u,13,3)||'-'||v||substr(u,17,3)||'-'||substr(u,21,12)
|
||||||
|
from (select lower(hex(randomblob(16))) as u, substr('89ab',abs(random()) % 4 + 1, 1) as v)")'"
|
||||||
|
messageid="$(sqlite3 "$history_db" "INSERT INTO messages(uid,thread_id,sender_id,body,body_type,direction,time,preview_id,status) VALUES(
|
||||||
|
$uid,
|
||||||
|
$threadid,
|
||||||
|
$sender_id,
|
||||||
|
$body,
|
||||||
|
$body_type,
|
||||||
|
$(( -(direction*2-3) )),
|
||||||
|
$date,
|
||||||
|
null,
|
||||||
|
$msg_status)
|
||||||
|
ON CONFLICT (uid,thread_id,body,time)
|
||||||
|
DO UPDATE SET status=$msg_status;
|
||||||
|
select id from messages where uid=$uid;")"
|
||||||
|
done <<<"$(sqlite3 "$database" "select _id, address, display_name, date/1000, type, status from sms order by cast(_id as integer);")"
|
||||||
|
|
||||||
|
# import MMS
|
||||||
|
while read msgdata; do
|
||||||
|
echo "$msgdata"
|
||||||
|
IFS='|' read -a msgdata_array <<<"$msgdata"
|
||||||
|
msgid="${msgdata_array[0]}"
|
||||||
|
uid="'${msgdata_array[1]}'"
|
||||||
|
date="${msgdata_array[2]}"
|
||||||
|
subject="${msgdata_array[3]:-}"
|
||||||
|
if [ -n "$subject" ]; then
|
||||||
|
subject="'$subject'"
|
||||||
|
else
|
||||||
|
subject=null
|
||||||
|
fi
|
||||||
|
IFS='|' read -a sender <<<$(sqlite3 "$database" "select address, display_name from sender where msg_id='$msgid';")
|
||||||
|
address="$(normalize_address "${sender[0]}")"
|
||||||
|
display_name="${sender[1]:-}"
|
||||||
|
users=()
|
||||||
|
if [ -z "$address" ]; then
|
||||||
|
direction=$CHATTY_DIRECTION_OUT
|
||||||
|
sender_id=null
|
||||||
|
msg_status="$MESSAGE_STATUS_DELIVERED"
|
||||||
|
while read line; do
|
||||||
|
IFS='|' read -a recipient <<<"$line"
|
||||||
|
users+=("$(normalize_address "${recipient[0]}")")
|
||||||
|
aliases["${recipient[0]}"]="${recipient[1]:-}"
|
||||||
|
done <<<$(sqlite3 "$database" "select address, display_name from recipient where msg_id='$msgid';")
|
||||||
|
else
|
||||||
|
# We don't collect recipient addresses for inbound messages
|
||||||
|
# They have little value, especially when you receive MMS sent to a whole address book
|
||||||
|
direction=$CHATTY_DIRECTION_IN
|
||||||
|
msg_status="$MESSAGE_STATUS_READ"
|
||||||
|
users=("$address")
|
||||||
|
aliases["$address"]="$display_name"
|
||||||
|
fi
|
||||||
|
thread_aliases=()
|
||||||
|
thread_members=()
|
||||||
|
for user in "${users[@]}"; do
|
||||||
|
thread_members+=("$(add_user "$user" "${aliases["$user"]:-}")")
|
||||||
|
thread_aliases+=("${aliases["$user"]:-$user}")
|
||||||
|
done
|
||||||
|
sender_id="${sender_id:-${thread_members[0]}}"
|
||||||
|
thread_name="'$(join_by "," "${users[@]}")'"
|
||||||
|
thread_alias="'$(join_by "," "${thread_aliases[@]}")'"
|
||||||
|
if [ "$thread_alias" = "$thread_name" ]; then
|
||||||
|
thread_alias=null
|
||||||
|
fi
|
||||||
|
local_user_accountid="$(sqlite3 "$history_db" "SELECT id from accounts where user_id=$MMS_SMS_USERID and protocol=$PROTOCOL_MMS_SMS;")"
|
||||||
|
if [ "${#users[@]}" -gt 1 ]; then
|
||||||
|
thread_type="$THREAD_GROUP_CHAT"
|
||||||
|
else
|
||||||
|
thread_type="$THREAD_DIRECT_CHAT"
|
||||||
|
fi
|
||||||
|
threadid="$(sqlite3 "$history_db" "INSERT INTO threads(name,alias,account_id,type,visibility) VALUES (
|
||||||
|
$thread_name,
|
||||||
|
$thread_alias,
|
||||||
|
$local_user_accountid,
|
||||||
|
$thread_type,
|
||||||
|
$THREAD_VISIBILITY_VISIBLE)
|
||||||
|
ON CONFLICT(name,account_id,type)
|
||||||
|
DO UPDATE SET alias=$thread_alias, visibility=$THREAD_VISIBILITY_VISIBLE;
|
||||||
|
select id from threads where name=$thread_name and account_id=$local_user_accountid and type=$thread_type;")"
|
||||||
|
for member in "${thread_members[@]}"; do
|
||||||
|
sqlite3 "$history_db" "INSERT OR IGNORE INTO thread_members(thread_id, user_id) VALUES (
|
||||||
|
$threadid,
|
||||||
|
$member);"
|
||||||
|
done
|
||||||
|
# Concat plain text parts as the body
|
||||||
|
body="$(sqlite3 "$database" "SELECT text FROM part where mid=$msgid and ct='text/plain' order by cast(_id as integer);" | sed "s/'/''/g")"
|
||||||
|
body="'${body:-}'"
|
||||||
|
body_type="$MESSAGE_TYPE_MMS"
|
||||||
|
# Create message record
|
||||||
|
messageid="$(sqlite3 "$history_db" "INSERT INTO messages(uid,thread_id,sender_id,body,body_type,direction,time,preview_id,status,subject) VALUES(
|
||||||
|
$uid,
|
||||||
|
$threadid,
|
||||||
|
$sender_id,
|
||||||
|
$body,
|
||||||
|
$body_type,
|
||||||
|
$(( -(direction*2-3) )),
|
||||||
|
$date,
|
||||||
|
null,
|
||||||
|
$msg_status,
|
||||||
|
$subject)
|
||||||
|
ON CONFLICT (uid,thread_id,body,time)
|
||||||
|
DO UPDATE SET status=$msg_status;
|
||||||
|
select id from messages where uid=$uid;")"
|
||||||
|
# Process binary attachements
|
||||||
|
parent="${XDG_DATA_HOME:-$HOME/.local/share}/chatty"
|
||||||
|
attachments_info="$(sqlite3 "$database" "SELECT ct, savepath FROM part where mid=$msgid and lower(ct) not in ('application/smil', 'text/plain') order by cast(_id as integer);")"
|
||||||
|
[ -z "$attachments_info" ] || while read line; do
|
||||||
|
IFS='|' read -a attachment <<<"$line"
|
||||||
|
mime_type="${attachment[0]}"
|
||||||
|
savepath="${attachment[1]}"
|
||||||
|
name="'$(basename "$savepath")'"
|
||||||
|
url="'file://$(urlencode -m "$savepath")'"
|
||||||
|
path="'${savepath#$parent/}'"
|
||||||
|
size="$(stat --printf="%s" "$savepath")"
|
||||||
|
mime_type_id="$(sqlite3 "$history_db" "INSERT OR IGNORE INTO mime_type(name) VALUES ('$mime_type');
|
||||||
|
select id from mime_type where name='$mime_type';")"
|
||||||
|
# File record
|
||||||
|
fileid="$(sqlite3 "$history_db" "INSERT INTO files(name,url,path,mime_type_id,size,status) VALUES (
|
||||||
|
$name,
|
||||||
|
$url,
|
||||||
|
$path,
|
||||||
|
$mime_type_id,
|
||||||
|
$size,
|
||||||
|
$CHATTY_FILE_DOWNLOADED)
|
||||||
|
ON CONFLICT(url)
|
||||||
|
DO UPDATE SET path=$path, size=$size, status=$CHATTY_FILE_DOWNLOADED;
|
||||||
|
select id from files where url=$url;")"
|
||||||
|
# File_metadata
|
||||||
|
if [[ "$mime_type" =~ ^(image|video|audio)/ ]]; then
|
||||||
|
raw_metadata="$( (ffmpeg -i "$savepath" 2<&1 || true) | grep -E '(Duration|Stream.* Video):')"
|
||||||
|
if [[ "$mime_type" =~ ^(image|video)/ ]]; then
|
||||||
|
if [[ "$raw_metadata" =~ \ ([0-9]+)x([0-9]+)\ ]]; then
|
||||||
|
width="${BASH_REMATCH[1]}"
|
||||||
|
height="${BASH_REMATCH[2]}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [[ "$mime_type" =~ ^(video|audio)/ ]]; then
|
||||||
|
if [[ "$raw_metadata" =~ Duration:\ ([0-9]+):([0-9]+):([0-9]+)(\.([0-9]+))?, ]]; then
|
||||||
|
duration=$((${BASH_REMATCH[1]}*3600+${BASH_REMATCH[2]}*60+${BASH_REMATCH[2]}))
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [ -n "${width:-}${height:-}${duration:-}" ]; then
|
||||||
|
width="${width:-null}"
|
||||||
|
height="${height:-null}"
|
||||||
|
duration="${duration:-null}"
|
||||||
|
sqlite3 "$history_db" "INSERT OR IGNORE INTO file_metadata(file_id,width,height,duration) VALUES (
|
||||||
|
$fileid,
|
||||||
|
$width,
|
||||||
|
$height,
|
||||||
|
$duration);"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
sqlite3 "$history_db" "INSERT OR IGNORE INTO message_files(message_id,file_id) VALUES (
|
||||||
|
$messageid,
|
||||||
|
$fileid);"
|
||||||
|
done <<<"$attachments_info"
|
||||||
|
done <<<"$(sqlite3 "$database" "select _id, m_id, date, sub from mms order by cast(mms._id as integer);")"
|
||||||
13
save_files
Executable file
13
save_files
Executable file
|
|
@ -0,0 +1,13 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
database="$1"
|
||||||
|
|
||||||
|
# Saving binary files only
|
||||||
|
# .smil parts are ignored (not used by Chatty)
|
||||||
|
# Plain text attachments will be converted into message bodies
|
||||||
|
# cf. chatty:src/mm/chatty-mmsd.c:chatty_mmsd_process_mms_message_attachments
|
||||||
|
while read path; do
|
||||||
|
mkdir -p "$(dirname "$path")"
|
||||||
|
sqlite3 "$database" "select binary_data from part where savepath='$path';" | base64 -d >"$path"
|
||||||
|
done <<<"$(sqlite3 "$database" "select savepath from part where binary_data is not null;")"
|
||||||
66
to_sqlite
Executable file
66
to_sqlite
Executable file
|
|
@ -0,0 +1,66 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
wkdir=
|
||||||
|
debug=
|
||||||
|
function cleanup () {
|
||||||
|
[ -z "$wkdir" -a -z "$debug" ] || rm -fr "$wkdir"
|
||||||
|
}
|
||||||
|
trap 'cleanup' EXIT
|
||||||
|
if [ "$1" = "-d" ]; then
|
||||||
|
debug=1
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
input_file=$(readlink -f "$1")
|
||||||
|
wkdir=$(mktemp -d)
|
||||||
|
cd "$wkdir"
|
||||||
|
|
||||||
|
# Generate one JSON file per table into the temporary directory
|
||||||
|
jq '[map(select(has("recipient_addresses")|not))[]]' "$input_file" >sms.json
|
||||||
|
jq '[map(select(has("recipient_addresses")))[] | del(.parts, .recipient_addresses, .sender_address)]' "$input_file" >mms.json
|
||||||
|
jq '[map(select(has("recipient_addresses")))[].sender_address]' "$input_file" >sender.json
|
||||||
|
jq '[map(select(has("recipient_addresses")))[].recipient_addresses[]]' "$input_file" >recipient.json
|
||||||
|
jq '[map(select(has("parts")))[].parts[]]' "$input_file" >part.json
|
||||||
|
|
||||||
|
# Tables creation
|
||||||
|
sms_keys=$(jq -r '[.[] | keys[]] | unique[]' sms.json)
|
||||||
|
mms_keys=$(jq -r '[.[] | keys[]] | unique[]' mms.json)
|
||||||
|
address_keys=$(jq -r '[.[] | keys[]] | unique[]' recipient.json)
|
||||||
|
part_keys=$(jq -r '[.[] | keys[]] | unique[]' part.json)
|
||||||
|
|
||||||
|
function create_table () {
|
||||||
|
name="$1"
|
||||||
|
shift
|
||||||
|
echo -n "CREATE TABLE $name("
|
||||||
|
echo -n $(echo "$@" | sed 's/ / TEXT,/g;s/$/ TEXT/')
|
||||||
|
echo ");"
|
||||||
|
}
|
||||||
|
|
||||||
|
create_table sms $sms_keys
|
||||||
|
create_table mms $mms_keys
|
||||||
|
create_table sender $address_keys
|
||||||
|
create_table recipient $address_keys
|
||||||
|
create_table part $part_keys savepath
|
||||||
|
|
||||||
|
# Populate
|
||||||
|
jq -r '.[] | to_entries | "INSERT INTO sms (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' sms.json
|
||||||
|
jq -r '.[] | to_entries | "INSERT INTO mms (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' mms.json
|
||||||
|
jq -r '.[] | to_entries | "INSERT INTO sender (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' sender.json
|
||||||
|
jq -r '.[] | to_entries | "INSERT INTO recipient (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' recipient.json
|
||||||
|
jq -r '.[] | to_entries | "INSERT INTO part (\([.[].key] | join(", "))) VALUES (\"\([.[].value | gsub("\""; "\"\"")] | join("\", \""))\");"' part.json
|
||||||
|
|
||||||
|
# Fix MMS out sender
|
||||||
|
echo "UPDATE sender set address=null where address in ('insert-address-token', 'Unknown address');"
|
||||||
|
|
||||||
|
# Delete MMS with no part
|
||||||
|
echo "DELETE FROM mms where not exists (SELECT 1 from part where part.mid=mms._id);"
|
||||||
|
|
||||||
|
# Build savepath for each part
|
||||||
|
# Part:
|
||||||
|
# date = mms.date
|
||||||
|
# address = sender.address || moi
|
||||||
|
# uid = cl ? cl : (cid sans les '<>')
|
||||||
|
# tag = date+address+uid
|
||||||
|
# savepath = $user_data_dir/chatty/mms/tag
|
||||||
|
user_data_dir="${XDG_DATA_HOME:-$HOME/.local/share}/chatty/mms"
|
||||||
|
echo "UPDATE part set savepath=(select '$user_data_dir/' || mms.date || (CASE WHEN sender.address IS NOT null THEN sender.address ELSE '' END) || mms.m_id || '/' || replace(replace(replace((CASE WHEN part.cl IS NOT null THEN part.cl ELSE trim(part.cid,'<>') END),'(','_'),')','_'),' ','_') from mms, sender where mms._id=part.mid and sender.msg_id=part.mid);"
|
||||||
Loading…
Add table
Reference in a new issue