reduced memory usage on POP3 session.

git-svn-id: svn://sylpheed.sraoss.jp/sylpheed/trunk@774 ee746299-78ed-0310-b773-934348b2243d

3 files changed, 311 insertions, 65 deletions

diff --git a/libsylph/pop.c b/libsylph/pop.c
index 7ee68af2..b61eb377 100644
--- a/libsylph/pop.c
+++ b/libsylph/pop.c
@@ -64,7 +64,7 @@ static gint pop3_getsize_list_recv	(Pop3Session *session,
 					 guint        len);
 static gint pop3_retr_send		(Pop3Session *session);
 static gint pop3_retr_recv		(Pop3Session *session,
-					 const gchar *data,
+					 FILE	     *fp,
 					 guint        len);
 static gint pop3_delete_send		(Pop3Session *session);
 static gint pop3_delete_recv		(Pop3Session *session);
@@ -76,7 +76,7 @@ static void pop3_gen_send		(Pop3Session	*session,
 static void pop3_session_destroy	(Session	*session);
 
 static gint pop3_write_msg_to_file	(const gchar	*file,
-					 const gchar	*data,
+					 FILE		*src_fp,
 					 guint		 len);
 
 static Pop3State pop3_lookup_next	(Pop3Session	*session);
@@ -88,6 +88,10 @@ static gint pop3_session_recv_msg		(Session	*session,
 static gint pop3_session_recv_data_finished	(Session	*session,
 						 guchar		*data,
 						 guint		 len);
+static gint pop3_session_recv_data_as_file_finished
+						(Session	*session,
+						 FILE		*fp,
+						 guint		 len);
 
 
 static gint pop3_greeting_recv(Pop3Session *session, const gchar *msg)
@@ -328,13 +332,13 @@ static gint pop3_retr_send(Pop3Session *session)
 	return PS_SUCCESS;
 }
 
-static gint pop3_retr_recv(Pop3Session *session, const gchar *data, guint len)
+static gint pop3_retr_recv(Pop3Session *session, FILE *fp, guint len)
 {
 	gchar *file;
 	gint drop_ok;
 
 	file = get_tmp_file();
-	if (pop3_write_msg_to_file(file, data, len) < 0) {
+	if (pop3_write_msg_to_file(file, fp, len) < 0) {
 		g_free(file);
 		session->error_val = PS_IOERR;
 		return -1;
@@ -411,8 +415,10 @@ Session *pop3_session_new(PrefsAccount *account)
 	SESSION(session)->type = SESSION_POP3;
 
 	SESSION(session)->recv_msg = pop3_session_recv_msg;
-	SESSION(session)->recv_data_finished = pop3_session_recv_data_finished;
 	SESSION(session)->send_data_finished = NULL;
+	SESSION(session)->recv_data_finished = pop3_session_recv_data_finished;
+	SESSION(session)->recv_data_as_file_finished =
+		pop3_session_recv_data_as_file_finished;
 
 	SESSION(session)->destroy = pop3_session_destroy;
 
@@ -522,11 +528,11 @@ gint pop3_write_uidl_list(Pop3Session *session)
 	return 0;
 }
 
-static gint pop3_write_msg_to_file(const gchar *file, const gchar *data,
-				   guint len)
+static gint pop3_write_msg_to_file(const gchar *file, FILE *src_fp, guint len)
 {
 	FILE *fp;
-	const gchar *prev, *cur;
+	gchar buf[BUFFSIZE];
+	gchar last_ch = '\0';
 
 	g_return_val_if_fail(file != NULL, -1);
 
@@ -538,55 +544,41 @@ static gint pop3_write_msg_to_file(const gchar *file, const gchar *data,
 	if (change_file_mode_rw(fp, file) < 0)
 		FILE_OP_ERROR(file, "chmod");
 
-	/* +------------------+----------------+--------------------------+ *
-	 * ^data              ^prev            ^cur             data+len-1^ */
+	while (fgets(buf, sizeof(buf), src_fp) != NULL) {
+		gchar *p = buf;
+		gint len;
+
+		len = strlen(buf);
+		if (len > 0) {
+			last_ch = buf[len - 1];
+			if (last_ch == '\n' && len > 1 &&
+			    buf[len - 2] == '\r') {
+				buf[len - 2] = '\n';
+				buf[len - 1] = '\0';
+			} else if (last_ch == '\r')
+				buf[len - 1] = '\0';
+		} else
+			last_ch = '\0';
 
-	prev = data;
-	while ((cur = (gchar *)my_memmem(prev, len - (prev - data), "\r\n", 2))
-	       != NULL) {
-		if ((cur > prev && fwrite(prev, cur - prev, 1, fp) < 1) ||
-		    fputc('\n', fp) == EOF) {
-			FILE_OP_ERROR(file, "fwrite");
+		if ((last_ch == '\0' || last_ch == '\n') &&
+		    *p == '.' && *(p + 1) == '.')
+			p++;
+
+		if (fputs(p, fp) == EOF) {
+			FILE_OP_ERROR(file, "fputs");
 			g_warning("can't write to file: %s\n", file);
 			fclose(fp);
 			g_unlink(file);
 			return -1;
 		}
-
-		if (cur == data + len - 1) {
-			prev = cur + 1;
-			break;
-		}
-
-		if (*(cur + 1) == '\n')
-			prev = cur + 2;
-		else
-			prev = cur + 1;
-
-		if (prev - data < len - 1 && *prev == '.' && *(prev + 1) == '.')
-			prev++;
-
-		if (prev - data >= len)
-			break;
 	}
 
-	if (prev - data < len &&
-	    fwrite(prev, len - (prev - data), 1, fp) < 1) {
-		FILE_OP_ERROR(file, "fwrite");
-		g_warning("can't write to file: %s\n", file);
+	if (ferror(src_fp)) {
+		FILE_OP_ERROR(file, "fgets");
 		fclose(fp);
 		g_unlink(file);
 		return -1;
 	}
-	if (data[len - 1] != '\r' && data[len - 1] != '\n') {
-		if (fputc('\n', fp) == EOF) {
-			FILE_OP_ERROR(file, "fputc");
-			g_warning("can't write to file: %s\n", file);
-			fclose(fp);
-			g_unlink(file);
-			return -1;
-		}
-	}
 
 	if (fclose(fp) == EOF) {
 		FILE_OP_ERROR(file, "fclose");
@@ -786,7 +778,7 @@ static gint pop3_session_recv_msg(Session *session, const gchar *msg)
 		break;
 	case POP3_RETR:
 		pop3_session->state = POP3_RETR_RECV;
-		session_recv_data(session, 0, ".\r\n");
+		session_recv_data_as_file(session, 0, ".\r\n");
 		break;
 	case POP3_DELETE:
 		pop3_delete_recv(pop3_session);
@@ -834,25 +826,6 @@ static gint pop3_session_recv_data_finished(Session *session, guchar *data,
 		} else
 			return -1;
 		break;
-	case POP3_RETR_RECV:
-		if (pop3_retr_recv(pop3_session, (gchar *)data, len) < 0)
-			return -1;
-
-		if (pop3_session->msg[pop3_session->cur_msg].recv_time
-		    == RECV_TIME_DELETE ||
-		    (pop3_session->ac_prefs->rmmail &&
-		     pop3_session->ac_prefs->msg_leave_time == 0 &&
-		     pop3_session->msg[pop3_session->cur_msg].recv_time
-		     != RECV_TIME_KEEP))
-			pop3_delete_send(pop3_session);
-		else if (pop3_session->cur_msg == pop3_session->count)
-			pop3_logout_send(pop3_session);
-		else {
-			pop3_session->cur_msg++;
-			if (pop3_lookup_next(pop3_session) == POP3_ERROR)
-				return -1;
-		}
-		break;
 	case POP3_ERROR:
 	default:
 		return -1;
@@ -860,3 +833,31 @@ static gint pop3_session_recv_data_finished(Session *session, guchar *data,
 
 	return 0;
 }
+
+static gint pop3_session_recv_data_as_file_finished(Session *session, FILE *fp,
+						    guint len)
+{
+	Pop3Session *pop3_session = POP3_SESSION(session);
+
+	g_return_val_if_fail(pop3_session->state == POP3_RETR_RECV, -1);
+
+	if (pop3_retr_recv(pop3_session, fp, len) < 0)
+		return -1;
+
+	if (pop3_session->msg[pop3_session->cur_msg].recv_time
+	    == RECV_TIME_DELETE ||
+	    (pop3_session->ac_prefs->rmmail &&
+	     pop3_session->ac_prefs->msg_leave_time == 0 &&
+	     pop3_session->msg[pop3_session->cur_msg].recv_time
+	     != RECV_TIME_KEEP))
+		pop3_delete_send(pop3_session);
+	else if (pop3_session->cur_msg == pop3_session->count)
+		pop3_logout_send(pop3_session);
+	else {
+		pop3_session->cur_msg++;
+		if (pop3_lookup_next(pop3_session) == POP3_ERROR)
+			return -1;
+	}
+
+	return 0;
+}
diff --git a/libsylph/session.c b/libsylph/session.c
index 7fcc744c..1ded2ee6 100644
--- a/libsylph/session.c
+++ b/libsylph/session.c
@@ -44,12 +44,19 @@ static gboolean session_timeout_cb	(gpointer	 data);
 static gboolean session_recv_msg_idle_cb	(gpointer	 data);
 static gboolean session_recv_data_idle_cb	(gpointer	 data);
 
+static gboolean session_recv_data_as_file_idle_cb	(gpointer	 data);
+
 static gboolean session_read_msg_cb	(SockInfo	*source,
 					 GIOCondition	 condition,
 					 gpointer	 data);
 static gboolean session_read_data_cb	(SockInfo	*source,
 					 GIOCondition	 condition,
 					 gpointer	 data);
+
+static gboolean session_read_data_as_file_cb	(SockInfo	*source,
+						 GIOCondition	 condition,
+						 gpointer	 data);
+
 static gboolean session_write_msg_cb	(SockInfo	*source,
 					 GIOCondition	 condition,
 					 gpointer	 data);
@@ -82,6 +89,12 @@ void session_init(Session *session)
 
 	session->read_msg_buf = g_string_sized_new(1024);
 	session->read_data_buf = g_byte_array_new();
+	session->read_data_terminator = NULL;
+
+	session->read_data_fp = NULL;
+	session->read_data_pos = 0;
+
+	session->preread_len = 0;
 
 	session->write_buf = NULL;
 	session->write_buf_p = NULL;
@@ -183,6 +196,8 @@ void session_destroy(Session *session)
 	g_string_free(session->read_msg_buf, TRUE);
 	g_byte_array_free(session->read_data_buf, TRUE);
 	g_free(session->read_data_terminator);
+	if (session->read_data_fp)
+		fclose(session->read_data_fp);
 	g_free(session->write_buf);
 
 	debug_print("session (%p): destroyed\n", session);
@@ -437,6 +452,49 @@ static gboolean session_recv_data_idle_cb(gpointer data)
 	return FALSE;
 }
 
+gint session_recv_data_as_file(Session *session, guint size,
+			       const gchar *terminator)
+{
+	g_return_val_if_fail(session->read_data_pos == 0, -1);
+	g_return_val_if_fail(session->read_data_fp == NULL, -1);
+
+	session->state = SESSION_RECV;
+
+	g_free(session->read_data_terminator);
+	session->read_data_terminator = g_strdup(terminator);
+	g_get_current_time(&session->tv_prev);
+
+	session->read_data_fp = my_tmpfile();
+	if (!session->read_data_fp) {
+		FILE_OP_ERROR("session_recv_data_as_file", "my_tmpfile");
+		return -1;
+	}
+
+	if (session->read_buf_len > 0)
+		g_idle_add(session_recv_data_as_file_idle_cb, session);
+	else
+		session->io_tag = sock_add_watch(session->sock, G_IO_IN,
+						 session_read_data_as_file_cb,
+						 session);
+
+	return 0;
+}
+
+static gboolean session_recv_data_as_file_idle_cb(gpointer data)
+{
+	Session *session = SESSION(data);
+	gboolean ret;
+
+	ret = session_read_data_cb(session->sock, G_IO_IN, session);
+
+	if (ret == TRUE)
+		session->io_tag = sock_add_watch(session->sock, G_IO_IN,
+						 session_read_data_as_file_cb,
+						 session);
+
+	return FALSE;
+}
+
 static gboolean session_read_msg_cb(SockInfo *source, GIOCondition condition,
 				    gpointer data)
 {
@@ -628,6 +686,176 @@ static gboolean session_read_data_cb(SockInfo *source, GIOCondition condition,
 	return FALSE;
 }
 
+#define READ_BUF_LEFT() \
+	(SESSION_BUFFSIZE - (session->read_buf_p - session->read_buf) - \
+	 session->read_buf_len)
+#define PREREAD_SIZE	8
+
+static gboolean session_read_data_as_file_cb(SockInfo *source,
+					     GIOCondition condition,
+					     gpointer data)
+{
+	Session *session = SESSION(data);
+	gint terminator_len;
+	gchar *data_begin_p;
+	gint buf_data_len;
+	gboolean complete = FALSE;
+	gint read_len;
+	gint write_len;
+	gint ret;
+
+	g_return_val_if_fail(condition == G_IO_IN, FALSE);
+
+	session_set_timeout(session, session->timeout_interval);
+
+	if (session->read_buf_len != 0)
+		g_print("already read %d bytes\n", session->read_buf_len);
+
+	if (session->read_buf_len == 0) {
+		read_len = sock_read(session->sock, session->read_buf_p,
+				     READ_BUF_LEFT());
+
+		if (read_len == 0) {
+			g_warning("sock_read: received EOF\n");
+			session->state = SESSION_EOF;
+			return FALSE;
+		}
+
+		if (read_len < 0) {
+			switch (errno) {
+			case EAGAIN:
+				return TRUE;
+			default:
+				g_warning("sock_read: %s\n", g_strerror(errno));
+				session->state = SESSION_ERROR;
+				return FALSE;
+			}
+		}
+
+		g_print("read %d bytes\n", read_len);
+		session->read_buf_len = read_len;
+	}
+
+	terminator_len = strlen(session->read_data_terminator);
+
+	if (session->read_buf_len == 0)
+		return TRUE;
+
+	/* +---------------buf_data_len---------------+
+	 * +--preread_len--+-------read_buf_len-------+
+	 * +---------------+--------------------------+-------------------+ *
+	 * ^data_begin_p   ^read_buf_p
+	 * ^read_buf
+	 */
+
+	data_begin_p = session->read_buf_p - session->preread_len;
+	buf_data_len = session->preread_len + session->read_buf_len;
+
+	/* check if data is terminated */
+	if (buf_data_len >= terminator_len) {
+		if (session->read_data_pos == 0 &&
+		    buf_data_len == terminator_len &&
+		    memcmp(data_begin_p, session->read_data_terminator,
+			   terminator_len) == 0)
+			complete = TRUE;
+		else if (buf_data_len >= terminator_len + 2 &&
+			 memcmp(data_begin_p + buf_data_len -
+				(terminator_len + 2), "\r\n", 2) == 0 &&
+			 memcmp(data_begin_p + buf_data_len -
+				terminator_len, session->read_data_terminator,
+				terminator_len) == 0)
+			complete = TRUE;
+	}
+
+	/* incomplete read */
+	if (!complete) {
+		GTimeVal tv_cur;
+
+		if (buf_data_len <= PREREAD_SIZE) {
+			if (data_begin_p > session->read_buf) {
+				g_memmove(session->read_buf, data_begin_p,
+					  buf_data_len);
+				session->read_buf_p =
+					session->read_buf + buf_data_len;
+			}
+			g_print("buffer data (%d) <= PREREAD_SIZE\n", buf_data_len);
+			session->preread_len = buf_data_len;
+			session->read_buf_len = 0;
+			return TRUE;
+		}
+
+		write_len = buf_data_len - PREREAD_SIZE;
+		g_print("write_len: %d\n", write_len);
+		if (fwrite(data_begin_p, write_len, 1,
+			   session->read_data_fp) < 1) {
+			g_warning("session_read_data_as_file_cb: "
+				  "writing data to file failed\n");
+			session->state = SESSION_ERROR;
+			return FALSE;
+		}
+		session->read_data_pos += write_len;
+		g_print("written %d bytes\n", session->read_data_pos);
+
+		g_memmove(session->read_buf, data_begin_p + write_len,
+			  PREREAD_SIZE);
+		session->read_buf_p = session->read_buf + PREREAD_SIZE;
+		session->preread_len = PREREAD_SIZE;
+		session->read_buf_len = 0;
+
+		g_get_current_time(&tv_cur);
+		if (tv_cur.tv_sec - session->tv_prev.tv_sec > 0 ||
+		    tv_cur.tv_usec - session->tv_prev.tv_usec >
+		    UI_REFRESH_INTERVAL) {
+			session->recv_data_progressive_notify
+				(session, session->read_data_pos, 0,
+				 session->recv_data_progressive_notify_data);
+			g_get_current_time(&session->tv_prev);
+		}
+
+		return TRUE;
+	}
+
+	/* complete */
+	g_print("data completed\n");
+	if (session->io_tag > 0) {
+		g_source_remove(session->io_tag);
+		session->io_tag = 0;
+	}
+
+	write_len = buf_data_len - terminator_len;
+	if (write_len > 0 && fwrite(data_begin_p, write_len, 1,
+				    session->read_data_fp) < 1) {
+		g_warning("session_read_data_as_file_cb: "
+			  "writing data to file failed\n");
+		session->state = SESSION_ERROR;
+		return FALSE;
+	}
+	session->read_data_pos += write_len;
+	g_print("total %d bytes\n\n", session->read_data_pos);
+
+	rewind(session->read_data_fp);
+
+	/* callback */
+	ret = session->recv_data_as_file_finished
+		(session, session->read_data_fp, session->read_data_pos);
+
+	fclose(session->read_data_fp);
+	session->read_data_fp = NULL;
+
+	session->recv_data_notify(session, session->read_data_pos,
+				  session->recv_data_notify_data);
+
+	session->read_data_pos = 0;
+	session->preread_len = 0;
+	session->read_buf_len = 0;
+	session->read_buf_p = session->read_buf;
+
+	if (ret < 0)
+		session->state = SESSION_ERROR;
+
+	return FALSE;
+}
+
 static gint session_write_buf(Session *session)
 {
 	gint write_len;
diff --git a/libsylph/session.h b/libsylph/session.h
index 988349d7..1bd948eb 100644
--- a/libsylph/session.h
+++ b/libsylph/session.h
@@ -111,10 +111,19 @@ struct _Session
 	gchar *read_buf_p;
 	gint read_buf_len;
 
+	/* buffer for short messages */
 	GString *read_msg_buf;
+
+	/* buffer for relatively short multiple lines data */
 	GByteArray *read_data_buf;
 	gchar *read_data_terminator;
 
+	/* buffer for large data */
+	FILE *read_data_fp;
+	gint read_data_pos;
+
+	gint preread_len;
+
 	/* buffer for short messages */
 	gchar *write_buf;
 	gchar *write_buf_p;
@@ -140,6 +149,10 @@ struct _Session
 					 guchar		*data,
 					 guint		 len);
 
+	gint (*recv_data_as_file_finished)	(Session	*session,
+						 FILE		*fp,
+						 guint		 len);
+
 	void (*destroy)			(Session	*session);
 
 	/* notification functions */
@@ -202,4 +215,8 @@ gint session_recv_data	(Session	*session,
 			 guint		 size,
 			 const gchar	*terminator);
 
+gint session_recv_data_as_file	(Session	*session,
+				 guint		 size,
+				 const gchar	*terminator);
+
 #endif /* __SESSION_H__ */