#!/usr/bin/perl

# Licence: GPL 2006, Samage
# Author: Mart van Santen <mart at samage dot net>
# Latest version on: http://samage.net/dev/
#
# This scrips converts maildir boxes to cyrus boxes, 
# please BE VERY CAREFULL WITH EXECUTION OF THIS SCRIPT,
# it is made for our internal use and some options 
# are a littlebit dangerous. 
#
# What it does is this:
#
# - Takes a Maildir box en copies its email to a cyrus box
# - Read flags from user Maildir boxes and write them in 
#   the cyrus database
# - Creates cyrus mailboxes
#
# Some code works on the cyrus databases directly, without
# using the cyrus commands. Please be very carefull if 
# you are running an other version then we did
# (that was, 2.2.13-8 from debian etch/testing)
# It is possible that the database format has been changed.
#
# Also be sure cyrus is not running when converting
#
# It alters these files directly (some files also indirectly):
# 1. /var/lib/cyrus/mailboxes.db (cyrus skiplist file)
# 2. /var/lib/cyrus/user/$char/$user.seen (skiplist file)
# 3. /var/lib/cyrus/user/$char/$user.sub  (plaintext file)
# 4. /var/spool/cyrus/mail/$char/user/$mailbox/cyrus.header
# 5. /var/spool/cyrus/mail/$char/user/$mailbox/cyrus.index
#
# This is what is done:
# For file 1. Add user mailboxes and subboxes
# For file 2. Write list of 'seen' messages
# For file 3. Write list of subscribed mailboxes
# For file 4. Read mailbox id & add custom flags
# For file 5. Write per message system & custom flags
#
# ALL FILES CREATED BY THIS SCRIPT ARE WORLD READABLE/WRITABLE,
# CHANGE FILE MODES AFTER CONVERSION
#
# Thanks to & parts used from:
# http://www.codepoets.co.uk/docs/courier2cyrus (David Goodwin)
# http://loophole.morpheus.net/linux/ (index_dump_pl.txt)
#
# This file is distributed under GPL, please send all 
# changes/additions to me.

# 
# usage: maildir2cyrus maildir_username cyrus_username
# 
# be sure cyrus is NOT running when executing


# Usefull include for debugging
use Data::Dumper;
use Date::Parse;
use Mail::Header;

# More output, use fast to 
$DEBUG = 1;
$FAST = 0;

$user = $ARGV[0];
$to = $ARGV[1];
$c = substr($to, 0, 1);


#####################################################
# Change these items according your local config

$source = "/mail/$user/Maildir";
$destination = "/mail/cyrus/mail/$c/user/$to";


$CYRUS_UID = 501;
$CYRUS_GID = 8;
$CYRUS_USER = "cyrus";
$CYRUS_RECONSTRUCT = "/usr/sbin/cyrreconstruct";
$CYRUS_DBCONVERT = "/usr/sbin/cvt_cyrusdb";
$CYRUS_ROOT = "/var/lib/cyrus";


# End: Local Config
#####################################################


print "Handling user: $user => $to\n";


# Some ugly global, used to map maildirfile to their
# cyrus index number
%map;


# Handel all
handleall($source, $destination, $to, $to);


# Main function, takes a userbox and convert it,
# also handle subfolders
sub handleall {
	my $src = $_[0];
	my $dst = $_[1];
	my $user = $_[2];
	my $user2 = $_[3];
	my %flags;

	my $c = 1;

	# Flush GLOBAL map
	undef(%map);
	$c = 1;
	$t = 1;

	print " * Processing mainfolder:\n" if $DEBUG;

	# Create userbox for this user
	createmb($user2, "");
	
	# Create destination dir
	$tmp = $dst;
	$tmp =~ s/'/'\\''/g;
	system("mkdir -p '$tmp'");
	chown $CYRUS_UID, $CYRUS_GID, $dst;
	chmod 0777, $dst;

	my ($uid_validity, $uid_last, $uid_map)=getuiddb($src);

	# Handele cur/new dirs of maildir
	$c = handledir("$src/cur",$dst, $c, $uid_map);
	$c = handledir("$src/new",$dst, $c, undef);

	# Reconstruct folder
	system("su $CYRUS_USER -c '$CYRUS_RECONSTRUCT user.$user'");

	# Add userflags
	$userflags = getuserflags($src);
	my $mailbox_id = handlecyrusheader($dst, $userflags);

	# Handle index: write flags
	handlecyrusindex($dst,$uid_validity,$c);

	# Handle seen database
	handlecyrusseen($mailbox_id, $user2);

	# Read subdits
	opendir(DIR, $src);
	@files = readdir(DIR);
	closedir(DIR);

	$t = ($c - 1);
	foreach $dir (@files) {
		next if $dir eq ".";
		next if $dir eq "..";
		next if (! -d "$src/$dir");
		next if (substr($dir, 0, 1) ne ".");
	
		print " * Processing subfolder: $dir\n" if $DEBUG;

		$c = 1;
		undef(%map);
		
		$src2 = $src . "/$dir";

		my ($uid_validity, $uid_last, $uid_map)=getuiddb($src2);

		$dir = substr($dir, 1);

		$dst_dir = $dir;
		$folder = $dir;
		$dst_dir =~ s/\./\//g;
		$dst2 = $dst . "/" . $dst_dir;

		$tmp = $dst2;
		$tmp =~ s/'/'\\''/g;
		system("mkdir -p '$tmp'");
		chmod 0777, $dst2;
		chown $CYRUS_UID, $CYRUS_GID, $dst2;


		createmb($user, $folder);


		$c = handledir("$src2/cur",$dst2, $c, $uid_map);
		$c = handledir("$src2/new",$dst2, $c, undef);
		$t += ($c - 1);
		$userflags = getuserflags($src2);

		$folder =~ s/'/'\\''/g;
		open(OUT, ">/tmp/cyrus-convert.sh");
		print OUT "#!/bin/sh\n";
		print OUT "$CYRUS_RECONSTRUCT 'user.$user.$folder'\n";
		close(OUT);
		chmod 0777, "/tmp/cyrus-convert.sh";

		system("su $CYRUS_USER -c /tmp/cyrus-convert.sh");

		my $mailbox_id = handlecyrusheader($dst2, $userflags);

		print "MAILBOX: $mailbox_id\n";
		handlecyrusindex($dst2,$uid_validity,$c);
		handlecyrusseen($mailbox_id, $user);

	}

	# New, add junk mailbox to every account
	createmb($user, "Spam");
	system("mkdir -p $dst/Spam");
	chmod 0777, "$dst/Spam";
	chown $CYRUS_UID, $CYRUS_GID, "$dst/Spam";
	open(OUT, ">/tmp/cyrus-convert.sh");
	print OUT "#!/bin/sh\n";
	print OUT "$CYRUS_RECONSTRUCT 'user.$user.Spam'\n";
	close(OUT);
	system("su $CYRUS_USER -c /tmp/cyrus-convert.sh");

	updatesubscriptions($user, $src);

	print " * Handled $t mails, finished\n";
}






sub updatesubscriptions {
	my $user = $_[0];
	my $src = $_[1];
	
	if (! -f "$src/courierimapsubscribed") {
		return;
	}

	print "  * Updating subscribtions\n" if $DEBUG;
	$l = substr($user, 0, 1);
	open(IN, "<$src/courierimapsubscribed");
	open(OUT, ">$CYRUS_ROOT/user/$l/${user}.sub");
	$junk = 0;
	while(<IN>) {
		$_ =~ s/^INBOX/user.$user/;
		$_ =~ s/$/\t/;
		print OUT $_;

		if ($_ =~ m/^user.$user.Spam/) {
			$junk = 1;
		}

	}
	if ($junk == 0) {
		print OUT "user.$user.Spam\t\n";
	}
	close(IN);
	close(OUT);
	chown $CYRUS_UID, $CYRUS_GID, "$CYRUS_ROOT/user/$l/${user}.sub";
	chmod 0666, "$CYRUS_ROOT/user/$l/${user}.sub";
	return;
	
}

sub createmb {
	my $user = $_[0];
	my $folder = $_[1];

	my $mb = $user;
	if ($folder ne "") {
		$mb .= "." . $folder;
	}
	unlink("/tmp/cyrus-mailbox");
	unlink("/tmp/cyrus-mailbox.out");
	system("$CYRUS_DBCONVERT $CYRUS_ROOT/mailboxes.db skiplist /tmp/cyrus-mailbox flat");
	open(IN, "</tmp/cyrus-mailbox");
	open(OUT, ">/tmp/cyrus-mailbox.out");
	$found = 0;
	while(<IN>) {
		if (m/^user.$mb\t/) {
			$found = 1;
		}
		print OUT $_;
	}
	if ($found == 0) {
		print "  - Add mailbox $mb\n" if $DEBUG;
		print OUT "user.$mb\t0 default $user\tlrswipcda\t\n";
	}
	close(IN);
	close(OUT);

	if ($found == 0) {
		unlink("$CYRUS_ROOT/mailboxes.db");
		system("$CYRUS_DBCONVERT /tmp/cyrus-mailbox.out flat $CYRUS_ROOT/mailboxes.db skiplist");
		chown $CYRUS_UID, $CYRUS_GID, "$CYRUS_ROOT/mailboxes.db";
		chmod 0666, "$CYRUS_ROOT/mailboxes.db";
	}

}





sub handlecyrusseen {
	my $mailbox = $_[0];
	my $user = $_[1];

	print "  - Fixing seen index file ($user)...\n" if $DEBUG;

	my @seen;
	foreach $file_id (keys %map) {
		if ($map{$file_id}{'systemflags'}{'seen'}) {
			push @seen, $map{$file_id}{'cyrusid'};
		}
	}
	@seen = sort {$a <=> $b} @seen;

	$seen_string = "";
	$last = -1;
	$range = 0;
	foreach $cyrus_id (@seen) {
		if ($cyrus_id == $last + 1) {
			$range = 1;
		}
		else {
			if ($range == 1) {
				$seen_string .= ":$last";
			}
			$range = 0;
			$seen_string .= ",$cyrus_id";
		}

		$last = $cyrus_id;

	}
	if ($range == 1) {
		$seen_string .= ":$last";
	}
	
	if ($seen_string eq "") {
		print "    - Empty mailbox, skipping\n" if $DEBUG;
		return;
	}

	$seen_string = substr($seen_string, 1);
	$stamp = time;

	$line = "$mailbox\t1 $stamp $last $stamp $seen_string";
	

	open(OUT, ">/tmp/cyrus-seen.out");


	$l = substr($user, 0, 1);
	$src = "$CYRUS_ROOT/user/$l/$user.seen";
	unlink("/tmp/cyrus-seen");
	my $written = 0;
	if (-f $src) {
		system("$CYRUS_DBCONVERT $src skiplist /tmp/cyrus-seen flat");
		open(IN, "</tmp/cyrus-seen");
		while(<IN>) {

			if ($_ =~ m/^$mailbox/) {
				print OUT $line . "\n"; 
				$written = 1;
			}
			else {
				print OUT $_;
			}
		}
		close(IN);
	}
	if ($written == 0) { print OUT $line . "\n"; }
	close(OUT);
	system("$CYRUS_DBCONVERT /tmp/cyrus-seen.out flat $src skiplist");
	chown $CYRUS_UID, $CYRUS_GID, $src;
	chmod 0666, $src;


#	DB format:
#	<Version>SP<Last Read Time>SP<Last Read UID>SP<Last Change Time>SP<List of Read UIDs>	


}

sub handlecyrusindex {

	print "  - Fixing mailbox index file...\n" if $DEBUG;

	my $dst = $_[0];
	my $uid_validity = $_[1];
	my $uid_last = $_[2];

	my $index_file = $dst . '/cyrus.index';


	my @field_names = ( "uid", "internal date", "sent date", "size",
            "header size", "content offset", "cache offset", "last updated",
            "system flags", "user 1", "user 2", "user 3", "user 4", "unknown 1", "unknown 2");

	my %flags = ( "answered" => 1<<0, "flagged" => 1<<1, "deleted" => 1<<2, "draft" => 1<<3 );


	sysopen(IDX,$index_file, 2) or return "Failed to open index.\n";

	#define OFFSET_LAST_UID 28
	$data = pack("N1", $uid_last);
	die "unable to seek!" unless sysseek(IDX, 28, 0);
	die "unable to write!" unless syswrite(IDX, $data, 4);

	#define OFFSET_UIDVALIDITY 44
	$data = pack("N1", $uid_validity);
	die "unable to seek!" unless sysseek(IDX, 44, 0);
	die "unable to write!" unless syswrite(IDX, $data, 4);

	#  Skip header..
	return "unable to seek $!" unless sysseek(IDX,76,0);
	my $seek = 76;

	my $i = 1;
	my $r = sysread(IDX,$doi,60);
	while($r){
		# First 8, system fields,
		# Next 5, flags,
		# Last 2, unknown
		my @data = unpack("N8N5N2",$doi);
		my $j = 0;
		foreach $field_name (@field_names){
			$record{$field_name} = $data[$j];
			$j++;
		}

		#print "INTERNALDATE is $record{'internal date'}\n";
		# Seek position
		$record{'seek'} = $seek;
		$records{$record{'uid'}} = { %record };
		$seek += 60;

		$r = sysread(IDX,$doi,60);
		$i++;
	}

	# Index file read (cyrus) 
	
	foreach $file_id (keys %map) {
		$cyrus_id = $map{$file_id}{'cyrusid'};

		if ($records{$cyrus_id}) {
			$offset = $records{$cyrus_id}{'seek'} + 4;
			if ($map{$file_id}{'internal date'}) {
				$data = $map{$file_id}{'internal date'};
				# Write internal date
				$data = pack("N1", $data);
				die "unable to seek!" unless sysseek(IDX, $offset, 0);
				die "unable to write!" unless syswrite(IDX, $data, 4);
			} else {
				print "No internal date found for maildir file: " . $file_id . "; the conversion date will be used in some clients\n";
			}

			$offset = $records{$cyrus_id}{'seek'} + 32;
			$data = 0;

			$data = $data | 1<<0 if $map{$file_id}{'systemflags'}{'replied'};
			$data = $data | 1<<1 if $map{$file_id}{'systemflags'}{'flagged'};
#			$data = $data | 1<<2 if $map{$file_id}{'systemflags'}{'deleted'};
			$data = $data | 1<<3 if $map{$file_id}{'systemflags'}{'draft'};

			# Write system flags
			$data = pack("N1", $data);
			die "unable to seek!" unless sysseek(IDX, $offset, 0);
			die "unable to write!" unless syswrite(IDX, $data, 4);

			# Write custom flags
			$flag_string = $map{$file_id}{'customflags'};
			@custom_flags = split /\ /, $flag_string;
			
			$data = 0;
			foreach $flag (@custom_flags) {
				$data = $data | 1<<$flag
			}

			$data = pack("N4", $data);
			die "unable to seek!" unless sysseek(IDX, $offset + 4, 0);
			die "unable to write!" unless syswrite(IDX, $data, 16);


		}
		else {
			print "No cyrus-index entry found for maildir file: " . $file_id . "\n";
		}

	}


	close(IDX);
}

sub handlecyrusheader {
	my $dst = $_[0];
	my $customflags = $_[1];

	my $header = $dst . '/cyrus.header';

	print "  - Fixing mailbox header file...\n" if $DEBUG;

	open(IN, "<$header");
	$data = "";
	while(<IN>) {
		$data .= $_;
	}
	close(IN);
	
	# Header = 115 chars
	# Header + tab = 116
	my $offset = 116;
	my $eol = index($data, "\n",$offset);
	my $neol = index($data, "\n",$eol+1);

	my $mailbox = substr($data, $offset, $eol - $offset);

	my $start = substr($data, 0, $eol+1);
	my $end = substr($data, $neol);

	open(OUT, ">$header");
	print OUT $start;
	foreach my $key (@{$customflags}) {
		print OUT "$key ";
	}
	print OUT $end;
	close(OUT);

	return $mailbox;
}





sub getuserflags {
	my $src = $_[0];
	my @flags;
	return \@flags if (! -d "$src/courierimapkeywords");
	return \@flags if (! -f "$src/courierimapkeywords/:list");

	open(IN, "<$src/courierimapkeywords/:list") or die ("Error opening");

	my $header_done = 0;
	my $k = 0;
	while(<IN>) {
		if ($_ eq "\n") { $header_done = 1; next; };
		chop;

		if ($header_done == 0) {
			$flags[$k] = $_;
			$k++;
		}
		else {
			
			my $file_id_pos = index($_, ":");
			my $file_id = substr($_, 0, $file_id_pos);
		
			# Check if file is in map, else no need to add to index
			if ($map{$file_id}) {
				$flag_string = substr($_, $file_id_pos + 1);
				$map{$file_id}{'customflags'} = $flag_string;
			}

		}

	}
	close(IN);

	return \@flags;
}


sub getuiddb {
	my $src = $_[0];

	my $version;
	my %uid_map;
	my $uid_validity;
	my $uid_last;
	my $uid_next;

	return undef if (! -f "$src/courierimapuiddb");
	open(IN, "<$src/courierimapuiddb") or die ("Error opening");

	# 1 1089373752 890
	($version,$uid_validity,$uid_next)=split(/ /,<IN>);
	$uid_last=($uid_next>1)?($uid_next-1):0;
	printf("Maildir: %s, uid_validity=%d, uid_next=%d, uid_last=%d\n",$src,$uid_validity,$uid_next,$uid_last);

	# 577 1161107238.V39704I486200.fqdn.example.org
	# 578 1161107816.V39704I486201.fqdn.example.org
	while(<IN>) {
		chomp($_);
		my ($uid,$filename) = split(/ /);
		#printf("uid=%d, filename=%s\n", $uid,$filename);
		$uid_map{$filename}=$uid;
		if($uid>$uid_last) {
			$uid_last=$uid;
		}
	}
	close(IN);

	return ($uid_validity, $uid_last, \%uid_map);
}

sub handledir {
	my $src = $_[0];
	my $dst = $_[1];
	my $i = $_[2];
	my $uid_map = $_[3];
	my $file;
	$j = 0;
	if (! -d $src) { return $i; }
	opendir(DIR, $src);
	my @files = readdir(DIR);
	closedir(DIR);

	foreach $file (@files) {
		next if $file eq ".";
		next if $file eq "..";
		$j++;
		next if ($FAST && $j > 25);
		if (-f "$src/$file") {
			$file_src = "$src/$file";

			my $file_id_pos = index($file, ":");
			my $file_id = ($file_id_pos)?substr($file, 0, $file_id_pos):$file;
			$i=$uid_map->{$file_id} if defined($uid_map->{$file_id});
			#printf("file_id=%s, uid=%d\n", $file_id, $i);

			$file_dst = $dst . "/" . $i . ".";

			# If file already exists, increase number
			while (-f $file_dst) {
				$i++;	
				$file_dst = $dst . "/" . $i . ".";
			}
			my $file_id_pos = index($file, ":2,");
			my $file_id = substr($file, 0, $file_id_pos);
			$map{$file_id}{'cyrusid'} = $i;

			open(IN, "<$file_src");
			$searchdate = 1;
			$data = "";
			while(<IN>) {
				#$data .= $_;
				$line = $_;
                                $data .= $line;
                                if ($searchdate) {
                                        if ($line =~ /^Date:/i) {
                                                #print "Regel met datum gevonden: $line\n";
                                                $line =~ s/Date://i;
                                                $line =~ s/\n//i;
                                                #print "overgebleven regel is [$line]\nstr2time geeft: ";
                                                if (str2time($line)) {
                                                        $map{$file_id}{'internal date'} = str2time($line);
                                                        $searchdate = 0;
                                                } else {
                                                  # specific dutch replacements
                                                  # remove the week day
                                                  $line =~ s/^[^,]+,//;
                                                  # some months have difference abbr in dutch
                                                  $line =~ s/mrt/mar/i;
                                                  $line =~ s/mei/may/i;
                                                  $line =~ s/okt/oct/i;
                                                  $line =~ s/<br>//i;
                                                        if (str2time($line)) {
                                                                $map{$file_id}{'internal date'} = str2time($line);
                                                                $searchdate = 0;
                                                        }
                                                }
                                        }
                                }
                        }
                        if ($data !~ /\r\n/)    {
                        # messages with not yet crlf eol's
                         $data =~ s/\n/\r\n/g;
			}
			close(IN);
			open(OUT, ">$file_dst");
			print OUT $data;
			close(OUT);
			chown $CYRUS_UID, $CYRUS_GID, $file_dst;
			chmod 0666, $file_dst;

			
			#my $file_id_pos = index($file, ":2,");
			#my $file_id = substr($file, 0, $file_id_pos);
			#$map{$file_id}{'cyrusid'} = $i;

			$systemflags = getsystemflags($file_src);
			# TODO: Add dovecot custom flags

			$map{$file_id}{'cyrusid'} = $i;
			$map{$file_id}{'systemflags'} = $systemflags;


			$i++;
		}
		if ($i % 250 == 0) { print "Handled $i messages so far...\n" if $DEBUG; }
	}
	return $i;

}


sub getsystemflags {
	# R: replied
	# S: seen
	# T: deleted/junk
	# D: draft
	# F: flagged
	# P: Not used, implemented as user-flag

	my $file = $_[0];
	my $fs = index($file, ":2,");
	my $flag_string = substr($file, $fs+3);
	my %flags;
	$flags{'seen'} = 1 if index($flag_string, 'S') >= 0;
	$flags{'replied'} = 1 if index($flag_string, 'R') >= 0;
	$flags{'junk'} = 1 if index($flag_string, 'T') >= 0;
	$flags{'draft'} = 1 if index($flag_string, 'D') >= 0;
	$flags{'flagged'} = 1 if index($flag_string, 'F') >= 0;

	return \%flags;

}