Console Unicode fixes

Console Scintilla now set to UTF8, and UTF8 text used, stored and
set in the input editbox.  Printing unicode text to the console
still doesn't work properly, so it's not perfect, but it's much
better.

Included some instructions in the docs about working with unicode text.

Upped version to 0.9

Author: bruderstein

PythonScript/project/PythonScript2010.vcxproj

@@ -182,7 +182,7 @@
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <PreprocessorDefinitions>WIN32;NDEBUG;_WINDOWS;_USRDLL;PYTHONSCRIPT2010_EXPORTS;BOOST_PYTHON_STATIC_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
-      <AdditionalIncludeDirectories>../include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>../include;../res;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
     </ClCompile>
     <Link>

PythonScript/src/ConsoleDialog.cpp

@@ -7,6 +7,7 @@
 #include "Notepad_plus_msgs.h"
 #include "PluginInterface.h"
 #include "Docking.h"
+#include "WcharMbcsConverter.h"
 
 ConsoleDialog::ConsoleDialog() :
 	DockingDlgInterface(IDD_CONSOLE),
@@ -235,60 +236,65 @@ void ConsoleDialog::historyPrevious()
 {
     if (m_currentHistory > 0)
     {
-        char buffer[1000];
-        GetWindowTextA(m_hInput, buffer, 1000);
+		int length = GetWindowTextLength(m_hInput);
+        TCHAR *buffer = new TCHAR[length + 1];
+        GetWindowText(m_hInput, buffer, length + 1);
 
         // Not an empty string and different from orig
         if (buffer[0] && (m_historyIter == m_history.end() || *m_historyIter != buffer)) 
         {
             if (m_changes.find(m_currentHistory) == m_changes.end())
             {
-                m_changes.insert(std::pair<int, std::string>(m_currentHistory, std::string(buffer)));
+                m_changes.insert(std::pair<int, tstring>(m_currentHistory, tstring(buffer)));
             }
             else
             {
-                m_changes[m_currentHistory] = std::string(buffer);
+                m_changes[m_currentHistory] = tstring(buffer);
             }
         }
+		delete [] buffer;
 
         --m_currentHistory;
         --m_historyIter;
 
         // If there's no changes to the line, just copy the original
         if (m_changes.find(m_currentHistory) == m_changes.end())
         {
-            ::SetWindowTextA(m_hInput, m_historyIter->c_str());
+            ::SetWindowText(m_hInput, m_historyIter->c_str());
             ::SendMessage(m_hInput, EM_SETSEL, m_historyIter->size(), (LPARAM)m_historyIter->size());
         }
         else
         {
             // Set it as the changed string
-            ::SetWindowTextA(m_hInput, m_changes[m_currentHistory].c_str());
+            ::SetWindowText(m_hInput, m_changes[m_currentHistory].c_str());
             ::SendMessage(m_hInput, EM_SETSEL, m_changes[m_currentHistory].size(), (LPARAM)m_changes[m_currentHistory].size());
         }
+
     }
 }
 
 void ConsoleDialog::historyNext()
 {
     if (static_cast<size_t>(m_currentHistory) < m_history.size())
     {
-        char buffer[1000];
-        GetWindowTextA(m_hInput, buffer, 1000);
+        int length = GetWindowTextLength(m_hInput);
+        TCHAR *buffer = new TCHAR[length + 1];
+        GetWindowText(m_hInput, buffer, length + 1);
 
 
         // Not an empty string and different from orig
         if (buffer[0] && *m_historyIter != buffer) 
         {
             if (m_changes.find(m_currentHistory) == m_changes.end())
             {
-                m_changes.insert(std::pair<int, std::string>(m_currentHistory, std::string(buffer)));
+                m_changes.insert(std::pair<int, tstring>(m_currentHistory, tstring(buffer)));
             }
             else
             {
-                m_changes[m_currentHistory] = std::string(buffer);
+                m_changes[m_currentHistory] = tstring(buffer);
             }
         }
+		delete [] buffer;
 
         ++m_currentHistory;
         ++m_historyIter;
@@ -298,7 +304,7 @@ void ConsoleDialog::historyNext()
         {
             if (m_historyIter != m_history.end())
             {
-                ::SetWindowTextA(m_hInput, m_historyIter->c_str());
+                ::SetWindowText(m_hInput, m_historyIter->c_str());
                 ::SendMessage(m_hInput, EM_SETSEL, m_historyIter->size(), (LPARAM)m_historyIter->size());
             }
             else
@@ -309,18 +315,18 @@ void ConsoleDialog::historyNext()
         else
         {
             // Set it as the changed string
-            ::SetWindowTextA(m_hInput, m_changes[m_currentHistory].c_str());
+            ::SetWindowText(m_hInput, m_changes[m_currentHistory].c_str());
             ::SendMessage(m_hInput, EM_SETSEL, m_changes[m_currentHistory].size(), (LPARAM)m_changes[m_currentHistory].size());
         }
     }
 }
 
 
-void ConsoleDialog::historyAdd(const char *line)
+void ConsoleDialog::historyAdd(const TCHAR *line)
 {
     if (line && line[0])
     {
-        m_history.push_back(std::string(line));
+        m_history.push_back(tstring(line));
         m_currentHistory = m_history.size();
     }
 
@@ -332,7 +338,7 @@ void ConsoleDialog::historyEnd()
 {
     m_currentHistory = m_history.size();
     m_historyIter = m_history.end();
-    ::SetWindowTextA(m_hInput, "");
+    ::SetWindowText(m_hInput, _T(""));
 }
 
 
@@ -387,14 +393,20 @@ void ConsoleDialog::runStatement()
 	assert(m_console != NULL);
 	if (m_console)
 	{
-		char buffer[1000];
-		GetWindowTextA(::GetDlgItem(_hSelf, IDC_INPUT), buffer, 1000);
+		
+		HWND hText = ::GetDlgItem(_hSelf, IDC_INPUT);
+		int length = GetWindowTextLengthW(hText);
+		TCHAR *buffer = new TCHAR[length + 1];
+		GetWindowTextW(hText, buffer, length + 1);
 		historyAdd(buffer);
+		std::shared_ptr<char> charBuffer = WcharMbcsConverter::tchar2char(buffer);
+		delete [] buffer;
+
 		writeText(m_prompt.size(), m_prompt.c_str());
-		writeText(strlen(buffer), buffer);
+		writeText(strlen(charBuffer.get()), charBuffer.get());
 		writeText(1, "\n");
-		SetWindowTextA(::GetDlgItem(_hSelf, IDC_INPUT), "");
-		m_console->runStatement(buffer);
+		SetWindowText(hText, _T(""));
+		m_console->runStatement(charBuffer.get());
 	}
 }
 
@@ -428,6 +440,8 @@ void ConsoleDialog::createOutputWindow(HWND hParentWindow)
 	 *       ... to be continued
 	 */
 
+	// Set the codepage to UTF-8
+	callScintilla(SCI_SETCODEPAGE, 65001);
 
 	// 0 is stdout, black text
     callScintilla(SCI_STYLESETSIZE, 0 /* = style number */, 8 /* = size in points */);   

PythonScript/src/ConsoleDialog.h

@@ -45,7 +45,7 @@ class ConsoleDialog : public DockingDlgInterface
 	static LRESULT inputWndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam);
 	void historyNext();
 	void historyPrevious();
-	void historyAdd(const char *line);
+	void historyAdd(const TCHAR *line);
 	void historyEnd();
 
 	LRESULT callScintilla(UINT message, WPARAM wParam = 0, LPARAM lParam = 0)
@@ -72,9 +72,9 @@ class ConsoleDialog : public DockingDlgInterface
 	WNDPROC m_originalInputWndProc;
 	HICON m_hTabIcon;
 
-	std::list<std::string> m_history;
-	std::list<std::string>::iterator m_historyIter;
-	std::map<idx_t, std::string> m_changes;
+	std::list<tstring> m_history;
+	std::list<tstring>::iterator m_historyIter;
+	std::map<idx_t, tstring> m_changes;
 	idx_t m_currentHistory;
 	bool m_runButtonIsRun;
 

PythonScript/src/PythonScriptVersion.h

@@ -1,7 +1,7 @@
 #ifndef _PYTHONSCRIPTVERSION_H
 #define _PYTHONSCRIPTVERSION_H
 
-#define PYSCR_VERSION_NUMERIC 0,8,0,1
-#define PYSCR_VERSION_STRING "0.8.0.1"
+#define PYSCR_VERSION_NUMERIC 0,9,0,0
+#define PYSCR_VERSION_STRING "0.9.0.0"
 
 #endif

docs/source/intro.rst

@@ -62,6 +62,40 @@ If you create a new module (i.e. a new file), and want to use the functions defi
 As the startup script runs in the same namespace as the scripts (__main__), you don't need to import the Npp module in the scripts.
 
 
+Working with Unicode Text
+=========================
+
+Python 2.x works with multi-byte strings, as does Scintilla.  That means in most cases, you don't need to do anything special to deal with unicode data,
+as both sides are talking the same language.  However, there are a few things to observe, and occassionaly you'll need to do something special to achieve
+what you want to do.  One important point is to make sure your script is saved in the same encoding as your target file(s) - this helps unicode strings 
+be interpreted the same way. 
+
+If you need to work with the string (for instance chance the case), you need to convert the string to a Python Unicode string.  To convert the string
+append ``.decode('utf8')`` to the string. Obviously if your string is in a different format, use the name of the correct encoding.
+
+To put text back to Scintilla (so editor.something()), use .encode('utf8') from a unicode string.
+
+For example::
+	
+	# define a unicode variable
+	someUnicodeString = u'This häs fünny ünicode chäractêrs in it'
+	
+	# append the text to the current buffer - assuming the current buffer is set to utf8
+	editor.addText(someUnicodeString.encode('utf8')
+
+	# grab the first line
+	firstLine = editor.getLine(0)
+	
+	# convert it to unicode
+	firstLineUnicode = firstLine.decode('utf8')
+	
+	# make it upper case
+	firstLineUnicode = firstLineUnicode.upper()
+	
+	# and put the line back
+	editor.replaceWholeLine(firstLineUnicode.encode('utf8')
+	
+
 .. _Notifications:
 
 Handling Notifications
@@ -174,7 +208,9 @@ automatically, you don't need to do anything different) - the only difference is
 you might receive the event some time after it has actually occurred.  In normal circumstances the time delay is so small it
 doesn't matter, but you may need to be aware of it if you're doing something time-sensitive.  
 
-If this causes anyone a problem, please let me know, and I'll certainly see what I can do - but no promises!
+This does mean that cancelling the SCN_AUTOCSELECTION notification is not possible (as the SCI_AUTOCCANCEL message must be sent 
+before the notification returns).  I'm open to suggestions for this, but processing the notifications sychronously would mean 
+(slightly) reduced performance of "normal" code, added complication and more risk of obscure threading bugs.
 
 
 .. _Python: http://www.python.org/

todo.txt

@@ -1,6 +1,14 @@
 
 TODO: Stuff
-
+
+DONE EnsureDirectoryExists for user scripts on New Script
+
+
+
+
+
+
+--------------
 DONE - Console buffer.write writes '0' at the end?
 
 DONE console stdout for console commands.