From 6fc5e8accca0264e0e541e3686f71888e4b879a7 Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Sat, 2 Sep 2017 23:21:11 -0700 Subject: [PATCH 01/12] enable decompiler on windows I'm not sure why the decompiler was disabled on windows, it works just fine. The only reason I can think of is that it requires nokogiri which is a C extension, but the dep on nokogiri isn't exempted, so windows users who did `gem install twee2` were getting it anyway. Additionally, there are precompiled versions of nokogiri available for windows now, so you don't need a C compiler. Furthermore, RubyInstaller can install the C compilers now. --- lib/twee2.rb | 12 +++--- lib/twee2/decompiler.rb | 90 ++++++++++++++++++++--------------------- 2 files changed, 49 insertions(+), 53 deletions(-) diff --git a/lib/twee2.rb b/lib/twee2.rb index 3d83b3a..c48dbfc 100755 --- a/lib/twee2.rb +++ b/lib/twee2.rb @@ -74,14 +74,12 @@ def self.version_check end end - unless Gem.win_platform? - # Reverse-engineers a Twee2/Twine 2 output HTML file into a Twee2 source file - def self.decompile(url, output) - File::open(output, 'w') do |out| - out.print Decompiler::decompile(url) - end - puts "Done" + # Reverse-engineers a Twee2/Twine 2 output HTML file into a Twee2 source file + def self.decompile(url, output) + File::open(output, 'w') do |out| + out.print Decompiler::decompile(url) end + puts "Done" end def self.help diff --git a/lib/twee2/decompiler.rb b/lib/twee2/decompiler.rb index 7ee1393..489350f 100644 --- a/lib/twee2/decompiler.rb +++ b/lib/twee2/decompiler.rb @@ -1,54 +1,52 @@ -unless Gem.win_platform? - require 'rubygems' - require 'open-uri' - require 'nokogiri' +require 'rubygems' +require 'open-uri' +require 'nokogiri' - module Twee2 - class DecompilationFailedException < Exception; end +module Twee2 + class DecompilationFailedException < Exception; end - class Decompiler - def self.decompile(url) - result = '' - # Load the compiled HTML and sanity-check it - html = Nokogiri::HTML(open(url)) - raise(DecompilationFailedException, 'tw-storydata not found') unless storydata = html.at_css('tw-storydata') - # Extract the tw-storydata#name (StoryTitle) and #startnode - result << "::StoryTitle\n#{storydata[:name].strip}\n\n" - startnode_pid, startnode_name = storydata[:startnode].strip, nil - # Extract the custom CSS and Javascript, if applicable - if (css = storydata.at_css('#twine-user-stylesheet')) && ((css_content = css.content.strip) != '') - result << "::StoryCSS [stylesheet]\n#{css_content}\n\n" - end - if (js = storydata.at_css('#twine-user-script')) && ((js_content = js.content.strip) != '') - result << "::StoryJS [script]\n#{js.content}\n\n" - end - # Extract each passage - storydata.css('tw-passagedata').each do |passagedata| - # Check if this is the start passage and record this accordingly - startnode_name = passagedata[:name] if(startnode_pid == passagedata[:pid]) - # Write the passage out - result << "::#{passagedata[:name].strip}" - result << " [#{passagedata[:tags].strip}]" if passagedata[:tags].strip != '' - result << " <#{passagedata[:position].strip}>" if passagedata[:position].strip != '' - result << "\n#{tidyup_passagedata(passagedata.content.strip)}\n\n" - end - # Write the Twee2 settings out (compatability layer) - result << "::Twee2Settings [twee2]\n" - result << "@story_start_name = '#{startnode_name.gsub("'", "\\'")}'\n" if startnode_name - result << "\n" - # Return the result - result + class Decompiler + def self.decompile(url) + result = '' + # Load the compiled HTML and sanity-check it + html = Nokogiri::HTML(open(url)) + raise(DecompilationFailedException, 'tw-storydata not found') unless storydata = html.at_css('tw-storydata') + # Extract the tw-storydata#name (StoryTitle) and #startnode + result << "::StoryTitle\n#{storydata[:name].strip}\n\n" + startnode_pid, startnode_name = storydata[:startnode].strip, nil + # Extract the custom CSS and Javascript, if applicable + if (css = storydata.at_css('#twine-user-stylesheet')) && ((css_content = css.content.strip) != '') + result << "::StoryCSS [stylesheet]\n#{css_content}\n\n" end + if (js = storydata.at_css('#twine-user-script')) && ((js_content = js.content.strip) != '') + result << "::StoryJS [script]\n#{js.content}\n\n" + end + # Extract each passage + storydata.css('tw-passagedata').each do |passagedata| + # Check if this is the start passage and record this accordingly + startnode_name = passagedata[:name] if(startnode_pid == passagedata[:pid]) + # Write the passage out + result << "::#{passagedata[:name].strip}" + result << " [#{passagedata[:tags].strip}]" if passagedata[:tags].strip != '' + result << " <#{passagedata[:position].strip}>" if passagedata[:position].strip != '' + result << "\n#{tidyup_passagedata(passagedata.content.strip)}\n\n" + end + # Write the Twee2 settings out (compatability layer) + result << "::Twee2Settings [twee2]\n" + result << "@story_start_name = '#{startnode_name.gsub("'", "\\'")}'\n" if startnode_name + result << "\n" + # Return the result + result + end - protected + protected - # Fixes common problems with decompiled passage content - def self.tidyup_passagedata(passagedata_content) - passagedata_content.gsub(/\[\[ *(.*?) *\]\]/, '[[\1]]'). # remove excess spacing within links: not suitable for Twee-style source - gsub(/\[\[ *(.*?) *<- *(.*?) *\]\]/, '[[\1<-\2]]'). # ditto - gsub(/\[\[ *(.*?) *-> *(.*?) *\]\]/, '[[\1->\2]]'). # ditto - gsub(/\[\[ *(.*?) *\| *(.*?) *\]\]/, '[[\1|\2]]') # ditto - end + # Fixes common problems with decompiled passage content + def self.tidyup_passagedata(passagedata_content) + passagedata_content.gsub(/\[\[ *(.*?) *\]\]/, '[[\1]]'). # remove excess spacing within links: not suitable for Twee-style source + gsub(/\[\[ *(.*?) *<- *(.*?) *\]\]/, '[[\1<-\2]]'). # ditto + gsub(/\[\[ *(.*?) *-> *(.*?) *\]\]/, '[[\1->\2]]'). # ditto + gsub(/\[\[ *(.*?) *\| *(.*?) *\]\]/, '[[\1|\2]]') # ditto end end end \ No newline at end of file From cc546de5afb377281f88e2353dc977832154a92f Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Sat, 2 Sep 2017 22:04:36 -0700 Subject: [PATCH 02/12] get ifid and storyformat info during decompile --- lib/twee2/decompiler.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/twee2/decompiler.rb b/lib/twee2/decompiler.rb index 7ee1393..05bf5e7 100644 --- a/lib/twee2/decompiler.rb +++ b/lib/twee2/decompiler.rb @@ -15,6 +15,11 @@ def self.decompile(url) # Extract the tw-storydata#name (StoryTitle) and #startnode result << "::StoryTitle\n#{storydata[:name].strip}\n\n" startnode_pid, startnode_name = storydata[:startnode].strip, nil + ifid = storydata[:ifid] + story_format = storydata[:format] + if story_format == 'SugarCube' && storydata[:'format-version'].start_with?('2.') + story_format = 'SugarCube2' + end # Extract the custom CSS and Javascript, if applicable if (css = storydata.at_css('#twine-user-stylesheet')) && ((css_content = css.content.strip) != '') result << "::StoryCSS [stylesheet]\n#{css_content}\n\n" @@ -35,6 +40,8 @@ def self.decompile(url) # Write the Twee2 settings out (compatability layer) result << "::Twee2Settings [twee2]\n" result << "@story_start_name = '#{startnode_name.gsub("'", "\\'")}'\n" if startnode_name + result << "Twee2::build_config.story_ifid = '#{ifid}'\n" + result << "Twee2::build_config.story_format = '#{story_format}'\n" result << "\n" # Return the result result From 7b578e06d64e2b508596747767684c12dbc7ac8c Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Sun, 3 Sep 2017 00:33:18 -0700 Subject: [PATCH 03/12] Change default encoding to utf-8 -- this probably should not be hardcoded --- lib/twee2.rb | 4 ++-- lib/twee2/story_file.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/twee2.rb b/lib/twee2.rb index 3d83b3a..b3ea591 100755 --- a/lib/twee2.rb +++ b/lib/twee2.rb @@ -42,7 +42,7 @@ def self.build(input, output, options = {}) puts "::StoryIFID[twee2]\nTwee2::build_config.story_ifid = '#{build_config.story_ifid}'" end # Produce output file - File::open(output, 'w') do |out| + File::open(output, 'w', encoding: "utf-8") do |out| out.print build_config.story_format.compile end puts "Done" @@ -77,7 +77,7 @@ def self.version_check unless Gem.win_platform? # Reverse-engineers a Twee2/Twine 2 output HTML file into a Twee2 source file def self.decompile(url, output) - File::open(output, 'w') do |out| + File::open(output, 'w', encoding: "utf-8") do |out| out.print Decompiler::decompile(url) end puts "Done" diff --git a/lib/twee2/story_file.rb b/lib/twee2/story_file.rb index 00bfb1b..eb7986d 100644 --- a/lib/twee2/story_file.rb +++ b/lib/twee2/story_file.rb @@ -26,7 +26,7 @@ def initialize(filename) @child_story_files = [] # Load file into memory to begin with - lines = File::read(filename).split(/\r?\n/) + lines = File::read(filename, encoding: 'utf-8').split(/\r?\n/) # First pass - go through and perform 'includes' i, in_story_includes_section = 0, false while i < lines.length From 9506fa596dd5a10e262f2889134c1bb69ce7213e Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Sun, 3 Sep 2017 01:05:12 -0700 Subject: [PATCH 04/12] bump version to 0.6.0 --- lib/twee2/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/twee2/version.rb b/lib/twee2/version.rb index 4296e2f..4e24826 100755 --- a/lib/twee2/version.rb +++ b/lib/twee2/version.rb @@ -1,3 +1,3 @@ module Twee2 - VERSION = "0.5.0" + VERSION = "0.6.0" end From 4a840de4c24bae8ca64c586a759b530f4e76f757 Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Sun, 3 Sep 2017 01:05:36 -0700 Subject: [PATCH 05/12] fix docs to show decompile works on windows --- doc/usage.txt | 1 - web/source/documentation.html.haml | 2 +- web/source/install.html.haml | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/usage.txt b/doc/usage.txt index 747fbd8..0762e7b 100755 --- a/doc/usage.txt +++ b/doc/usage.txt @@ -16,7 +16,6 @@ Usage: twee2 decompile [URL] [output.tw2] Decompiles a Twee2/Twine 2 HTML output file at a specified URL into a Twee2 source file. - NOT AVAILABLE ON MICROSOFT WINDOWS. twee2 version Reports what version of Twee2 you're using, and checks what the diff --git a/web/source/documentation.html.haml b/web/source/documentation.html.haml index 13082b9..b4ce46d 100755 --- a/web/source/documentation.html.haml +++ b/web/source/documentation.html.haml @@ -403,7 +403,7 @@ title: Full documentation %p It's possible to convert existing (compiled) Twee2/Twine 2 story files, in HTML format, back into Twee2 source files for further editing. This can be used to convert your Twine 2 projects into Twee2 - files or to easily examine the contents of somebody else's story. This feature does not work on Microsoft + files or to easily examine the contents of somebody else's story. This feature now works on Microsoft Windows. To use it, run: %p %code twee2 decompile input.html output.tw2 diff --git a/web/source/install.html.haml b/web/source/install.html.haml index 61e73f5..6abf8e8 100644 --- a/web/source/install.html.haml +++ b/web/source/install.html.haml @@ -22,8 +22,6 @@ title: Install %h3 Windows %p %a{href: 'http://rubyinstaller.org/downloads/'} Download RubyInstaller - %p.small - Note that the 'decompile' feature, which turns Twine 2 story files into Twee2 source code, is not supported on Windows. .col-sm-4 .well From a6d53b94e150d4c1b1e61a53f913e9edd09eee6c Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Sun, 3 Sep 2017 02:25:04 -0700 Subject: [PATCH 06/12] encoding = utf8 more --- lib/twee2/story_file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/twee2/story_file.rb b/lib/twee2/story_file.rb index eb7986d..1eb9233 100644 --- a/lib/twee2/story_file.rb +++ b/lib/twee2/story_file.rb @@ -48,7 +48,7 @@ def initialize(filename) # include a file here because an @include directive was spotted prefix, filename = $1, $2.strip if File::exists?(filename) - lines[i,1] = File::read(filename).split(/\r?\n/).map{|l|"#{prefix}#{l}"} # insert in-place, with prefix of appropriate amount of whitespace + lines[i,1] = File::read(filename, encoding: 'utf-8').split(/\r?\n/).map{|l|"#{prefix}#{l}"} # insert in-place, with prefix of appropriate amount of whitespace i-=1 # process this line again, in case of ::@include nesting else puts "WARNING: tried to ::@include file '#{filename}' but file was not found." From e83f635acd59f468d11ed420bc1747bb40b30205 Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Sun, 3 Sep 2017 02:28:33 -0700 Subject: [PATCH 07/12] fix problem with backslash followed by single quote Ruby's gsub method is a little weird in how it handles substitution strings and there's no way to turn it off. I could escape the substitution string, but the translation is something like backslash followed by a number, & or single quote something like: def escape_sub(s) s.gsub(/\\[0-9&']/, '\\\&'); end this way strikes me as easier to reason about (though not necessarily any less convoluted) --- lib/twee2/story_file.rb | 2 +- lib/twee2/story_format.rb | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/twee2/story_file.rb b/lib/twee2/story_file.rb index 1eb9233..cbc38e7 100644 --- a/lib/twee2/story_file.rb +++ b/lib/twee2/story_file.rb @@ -114,7 +114,7 @@ def initialize(filename) # Returns the rendered XML that represents this story def xmldata data = @story_data.target! - data.gsub('{{STORY_JS}}', @story_js) + data.gsub('%', '%%').gsub('{{STORY_JS}}', '%{story_js}') % {story_js: @story_js} end # Runs HAML, Coffeescript etc. preprocessors across each applicable passage diff --git a/lib/twee2/story_format.rb b/lib/twee2/story_format.rb index 948d77b..58ceb87 100644 --- a/lib/twee2/story_format.rb +++ b/lib/twee2/story_format.rb @@ -16,7 +16,16 @@ def initialize(name) # Given a story file, injects it into the StoryFormat and returns the HTML results def compile - @source.gsub('{{STORY_NAME}}', Twee2::build_config.story_name).gsub('{{STORY_DATA}}', Twee2::build_config.story_file.xmldata).gsub('{{STORY_FORMAT}}', @name) + @source\ + .gsub('%', '%%') \ + .gsub('{{STORY_NAME}}', '%{STORY_NAME}') \ + .gsub('{{STORY_DATA}}', '%{STORY_DATA}') \ + .gsub('{{STORY_FORMAT}}', '%{STORY_FORMAT}') \ + % { + STORY_NAME: Twee2::build_config.story_name, + STORY_DATA: Twee2::build_config.story_file.xmldata, + STORY_FORMAT: @name, + } end # Returns an array containing the known StoryFormat names From 3ac65b17b66edb26a865832bd9f4190990ced78f Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Sun, 3 Sep 2017 02:41:09 -0700 Subject: [PATCH 08/12] on second thought, just use the block form of gsub. --- lib/twee2/story_file.rb | 2 +- lib/twee2/story_format.rb | 14 ++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/twee2/story_file.rb b/lib/twee2/story_file.rb index cbc38e7..ea25100 100644 --- a/lib/twee2/story_file.rb +++ b/lib/twee2/story_file.rb @@ -114,7 +114,7 @@ def initialize(filename) # Returns the rendered XML that represents this story def xmldata data = @story_data.target! - data.gsub('%', '%%').gsub('{{STORY_JS}}', '%{story_js}') % {story_js: @story_js} + data.gsub('{{STORY_JS}}') {@story_js} end # Runs HAML, Coffeescript etc. preprocessors across each applicable passage diff --git a/lib/twee2/story_format.rb b/lib/twee2/story_format.rb index 58ceb87..dbc3d9e 100644 --- a/lib/twee2/story_format.rb +++ b/lib/twee2/story_format.rb @@ -16,16 +16,10 @@ def initialize(name) # Given a story file, injects it into the StoryFormat and returns the HTML results def compile - @source\ - .gsub('%', '%%') \ - .gsub('{{STORY_NAME}}', '%{STORY_NAME}') \ - .gsub('{{STORY_DATA}}', '%{STORY_DATA}') \ - .gsub('{{STORY_FORMAT}}', '%{STORY_FORMAT}') \ - % { - STORY_NAME: Twee2::build_config.story_name, - STORY_DATA: Twee2::build_config.story_file.xmldata, - STORY_FORMAT: @name, - } + @source \ + .gsub('{{STORY_NAME}}') { Twee2::build_config.story_name } \ + .gsub('{{STORY_DATA}}') { Twee2::build_config.story_file.xmldata } \ + .gsub('{{STORY_FORMAT}}') { @name } end # Returns an array containing the known StoryFormat names From d8480c1d632483af84fe09e1f608596c704aa348 Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Sun, 3 Sep 2017 02:41:20 -0700 Subject: [PATCH 09/12] more utf8 --- lib/twee2/story_format.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/twee2/story_format.rb b/lib/twee2/story_format.rb index dbc3d9e..2bd65df 100644 --- a/lib/twee2/story_format.rb +++ b/lib/twee2/story_format.rb @@ -8,7 +8,7 @@ class StoryFormat def initialize(name) raise(StoryFormatNotFoundException) if !File::exists?(format_file_path = Twee2::buildpath("storyFormats/#{name}/format.js")) && !File::exists?(format_file_path = "#{name}/format.js") @name = name - format_file = File::read(format_file_path) + format_file = File::read(format_file_path, encoding: 'utf-8') format_data = format_file.match(/(["'])source\1 *: *(["']).*?[^\\]\2/)[0] format_data_for_json = "\{#{format_data}\}" @source = JSON.parse(format_data_for_json)['source'] From 07b8a49b758f4c53ec0372947404b92d19d25028 Mon Sep 17 00:00:00 2001 From: Mark Harviston Date: Fri, 15 Sep 2017 20:52:21 -0700 Subject: [PATCH 10/12] Address #29 create output directory if it does not exist --- lib/twee2.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/twee2.rb b/lib/twee2.rb index 1bd0151..30ce6c4 100755 --- a/lib/twee2.rb +++ b/lib/twee2.rb @@ -41,6 +41,8 @@ def self.build(input, output, options = {}) puts "NOTICE: You haven't specified your IFID. Consider adding to your code -" puts "::StoryIFID[twee2]\nTwee2::build_config.story_ifid = '#{build_config.story_ifid}'" end + # Make sure output directory exists + FileUtils.mkdir_p(File.dirname(output)) # Produce output file File::open(output, 'w', encoding: "utf-8") do |out| out.print build_config.story_format.compile From 41e1a6d32c17a2cd9a5cb7c1c526d901c06306d6 Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Mon, 19 Feb 2018 14:22:26 -0800 Subject: [PATCH 11/12] even more utf8 --- lib/twee2/decompiler.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/twee2/decompiler.rb b/lib/twee2/decompiler.rb index 82ce11b..8d427af 100644 --- a/lib/twee2/decompiler.rb +++ b/lib/twee2/decompiler.rb @@ -9,7 +9,7 @@ class Decompiler def self.decompile(url) result = '' # Load the compiled HTML and sanity-check it - html = Nokogiri::HTML(open(url)) + html = Nokogiri::HTML(open(url), nil, 'utf-8') raise(DecompilationFailedException, 'tw-storydata not found') unless storydata = html.at_css('tw-storydata') # Extract the tw-storydata#name (StoryTitle) and #startnode result << "::StoryTitle\n#{storydata[:name].strip}\n\n" From 2cf4cd4aed8b28451fdd2e603d84f6b312944b4d Mon Sep 17 00:00:00 2001 From: Mark Thomas West Harviston Date: Mon, 19 Feb 2018 14:22:38 -0800 Subject: [PATCH 12/12] debugging tools --- Gemfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index 30abbc9..e26e005 100755 --- a/Gemfile +++ b/Gemfile @@ -6,4 +6,9 @@ gem 'filewatcher' gem 'haml' gem 'coffee-script' gem 'nokogiri' -gem 'sass' \ No newline at end of file +gem 'sass' + +group :development do + gem 'twee2', path: '.' + gem 'pry-byebug' +end \ No newline at end of file